FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Packetization for RTP Payload Format For AV1 (v1.0)
3			* https://aomediacodec.github.io/av1-rtp-spec/
4			* Copyright (c) 2024 Axis Communications
5			*
6			* This file is part of FFmpeg.
7			*
8			* FFmpeg is free software; you can redistribute it and/or
9			* modify it under the terms of the GNU Lesser General Public
10			* License as published by the Free Software Foundation; either
11			* version 2.1 of the License, or (at your option) any later version.
12			*
13			* FFmpeg is distributed in the hope that it will be useful,
14			* but WITHOUT ANY WARRANTY; without even the implied warranty of
15			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16			* Lesser General Public License for more details.
17			*
18			* You should have received a copy of the GNU Lesser General Public
19			* License along with FFmpeg; if not, write to the Free Software
20			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21			*/
22
23			/**
24			* @file
25			* @brief AV1 / RTP packetization code (RTP Payload Format For AV1 (v1.0))
26			* @author Chris Hodges <chris.hodges@axis.com>
27			* @note This will remove TDs and OBU size fields
28			*/
29
30			#include "avformat.h"
31			#include "rtpenc.h"
32			#include "libavcodec/av1.h"
33			#include "rtp_av1.h"
34
35			// enable tracing of packet data
36			//#define RTPENC_AV1_VERBOSE_TRACE
37
38			// enable searching for sequence header as workaround for AV1 encoders
39			// that do not set AV_PKT_FLAG_KEY correctly
40			#define RTPENC_AV1_SEARCH_SEQ_HEADER 1
41
42		✗	void ff_rtp_send_av1(AVFormatContext *ctx, const uint8_t *frame_buf, int frame_size, int is_keyframe) {
43		✗	uint8_t aggr_hdr = 0;
44		✗	int last_packet_of_frame = 0;
45		✗	RTPMuxContext *rtp_ctx = ctx->priv_data;
46		✗	const uint8_t *obu_ptr = frame_buf;
47		✗	int start_new_packet = 0;
48		✗	unsigned int num_obus = 0;
49		✗	unsigned int rem_pkt_size = rtp_ctx->max_payload_size - 1;
50		✗	uint8_t *pkt_ptr = NULL;
51
52		✗	const uint8_t *curr_obu_ptr = NULL;
53		✗	uint32_t curr_elem_size = 0;
54		✗	int curr_obu_hdr = -1;
55		✗	int curr_obu_ext = -1;
56		✗	const uint8_t *last_obu_ptr = NULL;
57		✗	uint32_t last_elem_size = 0;
58		✗	int last_obu_hdr = -1;
59		✗	int last_obu_ext = -1;
60
61		✗	rtp_ctx->timestamp = rtp_ctx->cur_timestamp;
62
63			/* The payload structure is supposed to be straight-forward, but there are a
64			* couple of edge cases to be tackled and make things very complex.
65			* These are mainly due to:
66			* - the OBU element size being optional for the last element, but MANDATORY
67			* if there are more than 3 elements
68			* - the size field of the element is made up of a variable number of
69			* LEB bytes
70			* - the latter in combination with the desire to fill the max packet size
71			* could cause a catch22
72			* - if there's less than 2 bytes remaining (depending on the required LEB),
73			* one would not have space for the payload of an element and must instead
74			* start the next packet
75			* - if there's less than 3 bytes remaining, the header byte plus the
76			* optional extension byte will not fit in the fragment making the
77			* handling even more complicated
78			* - as some OBU types are supposed to be filtered out, it is hard to decide
79			* via the remaining length whether the outputted OBU element will
80			* actually be the last one
81			*
82			* There are two major ways to tackle that: Pre-parsing of all OBUs within a
83			* frame (adds memory complexity) or lazy copying of the prior element.
84			* Here, the latter is implemented.
85			*/
86
87		✗	if (is_keyframe) {
88			#if RTPENC_AV1_SEARCH_SEQ_HEADER
89			/* search for OBU_SEQUENCE_HEADER to get a better indication that
90			* the frame was marked as keyframe is really a KEY_FRAME and not
91			* a INTRA_ONLY frame. This might be unnecessary if the AV1 parser/
92			* encoder always correctly specifies AV_PKT_FLAG_KEY.
93			*
94			* Note: Spec does NOT prohibit resending bit-identical
95			* OBU_SEQUENCE_HEADER for ANY kind of frame, though!
96			*/
97		✗	int rem_size = frame_size;
98		✗	const uint8_t *buf_ptr = frame_buf;
99		✗	while (rem_size > 0) {
100			uint32_t obu_size;
101		✗	uint8_t obu_hdr = *buf_ptr++;
102		✗	uint8_t obu_type = (obu_hdr >> AV1S_OBU_TYPE) & AV1M_OBU_TYPE;
103			int num_lebs;
104
105		✗	if (obu_type == AV1_OBU_SEQUENCE_HEADER) {
106		✗	av_log(ctx, AV_LOG_DEBUG, "Marking FIRST packet\n");
107		✗	aggr_hdr |= AV1F_AGGR_HDR_FIRST_PKT;
108		✗	break;
109			}
110		✗	if (!(obu_hdr & AV1F_OBU_HAS_SIZE_FIELD)) {
111		✗	break;
112			}
113		✗	rem_size--;
114			// read out explicit OBU size
115		✗	num_lebs = parse_leb(ctx, buf_ptr, rem_size, &obu_size);
116		✗	if (!num_lebs) {
117		✗	break;
118			}
119		✗	buf_ptr += num_lebs;
120		✗	rem_size -= num_lebs;
121			// bound OBU payload against remaining data to avoid pointer/size
122			// wraparound (mirrors the check in the packetization loop below)
123		✗	if (obu_size > (uint32_t) rem_size) {
124		✗	break;
125			}
126		✗	buf_ptr += obu_size;
127		✗	rem_size -= obu_size;
128			}
129			#else // RTPENC_AV1_SEARCH_SEQ_HEADER
130			av_log(ctx, AV_LOG_DEBUG, "Marking FIRST packet\n");
131			aggr_hdr |= AV1F_AGGR_HDR_FIRST_PKT;
132			#endif // RTPENC_AV1_SEARCH_SEQ_HEADER
133			}
134		✗	rem_pkt_size = rtp_ctx->max_payload_size - 1;
135		✗	pkt_ptr = rtp_ctx->buf + 1;
136
137			#ifdef RTPENC_AV1_VERBOSE_TRACE
138			av_log(ctx, AV_LOG_TRACE, "AV1 Frame %d in (%x), size=%d:\n",
139			rtp_ctx->seq, rtp_ctx->flags, frame_size);
140			av_hex_dump_log(ctx, AV_LOG_TRACE, frame_buf, FFMIN(frame_size, 128));
141			#endif
142
143		✗	while (frame_size) {
144			uint32_t obu_size;
145		✗	int num_lebs = 0;
146		✗	int ext_byte = -1;
147
148		✗	uint8_t obu_hdr = *obu_ptr++;
149		✗	uint8_t obu_type = (obu_hdr >> AV1S_OBU_TYPE) & AV1M_OBU_TYPE;
150		✗	frame_size--;
151
152		✗	if (obu_hdr & AV1F_OBU_FORBIDDEN) {
153		✗	av_log(ctx, AV_LOG_ERROR, "Forbidden bit set in AV1 OBU header (0x%02x)\n", obu_hdr);
154		✗	return;
155			}
156
157		✗	if (obu_hdr & AV1F_OBU_EXTENSION_FLAG) {
158		✗	if (!frame_size) {
159		✗	av_log(ctx, AV_LOG_ERROR, "Out of data for AV1 OBU header extension byte\n");
160		✗	return;
161			}
162		✗	ext_byte = *obu_ptr++;
163		✗	frame_size--;
164			}
165
166		✗	if (obu_hdr & AV1F_OBU_HAS_SIZE_FIELD) {
167		✗	obu_hdr &= ~AV1F_OBU_HAS_SIZE_FIELD; // remove size field
168			// read out explicit OBU size
169		✗	num_lebs = parse_leb(ctx, obu_ptr, frame_size, &obu_size);
170		✗	if (num_lebs <= 0) {
171		✗	return;
172			}
173		✗	obu_ptr += num_lebs;
174		✗	frame_size -= num_lebs;
175			} else {
176		✗	av_log(ctx, AV_LOG_ERROR, "Cannot handle AV1 OBUs without size fields\n");
177		✗	return;
178			}
179
180		✗	if (obu_size > (unsigned) frame_size) {
181		✗	av_log(ctx, AV_LOG_ERROR, "AV1 OBU size %d larger than remaining frame size %d\n", obu_size, frame_size);
182		✗	return;
183			}
184
185		✗	if (obu_size > 0xfffffffd) {
186		✗	av_log(ctx, AV_LOG_ERROR, "AV1 OBU size 0x%x might overflow (attack?)\n", obu_size);
187		✗	return;
188			}
189
190		✗	frame_size -= obu_size;
191
192		✗	if ((obu_type == AV1_OBU_TEMPORAL_DELIMITER) ||
193		✗	(obu_type == AV1_OBU_TILE_LIST) ||
194			(obu_type == AV1_OBU_PADDING)) {
195			// ignore and remove according to spec (note that OBU_PADDING is not
196			// mentioned in spec, but it does not make sense to transmit it).
197		✗	obu_ptr += obu_size;
198			// additional handling if the ignored OBU was the last one
199		✗	if (!frame_size) {
200			// we're done, flush the last packet, set RTP marker bit
201		✗	last_packet_of_frame = 1;
202		✗	goto flush_last_packet;
203			}
204		✗	continue;
205			}
206
207			/* if the last OBU had a temporal or spatial ID, they need to match to
208			* current; otherwise start new packet */
209		✗	if ((last_obu_ext >= 0) && (curr_obu_ext != last_obu_ext)) {
210		✗	start_new_packet = 1;
211			}
212
213		✗	flush_last_packet:
214		✗	last_obu_ptr = curr_obu_ptr;
215		✗	last_elem_size = curr_elem_size;
216		✗	last_obu_hdr = curr_obu_hdr;
217		✗	last_obu_ext = curr_obu_ext;
218
219		✗	curr_obu_ptr = obu_ptr; // behind header
220		✗	curr_elem_size = obu_size + 1 + ((ext_byte >= 0) ? 1 : 0);
221		✗	curr_obu_hdr = obu_hdr;
222		✗	curr_obu_ext = ext_byte;
223
224		✗	obu_ptr += obu_size;
225
226		✗	if (last_obu_ptr) {
227		✗	unsigned int first_elem_with_size = last_elem_size + calc_leb_size(last_elem_size);
228			// check if last packet fits completely and has reasonable space for
229			// at least a fragment of the next
230		✗	if (!last_packet_of_frame && (first_elem_with_size + 10 < rem_pkt_size)) {
231		✗	num_lebs = write_leb(pkt_ptr, last_elem_size);
232		✗	pkt_ptr += num_lebs;
233		✗	rem_pkt_size -= num_lebs;
234			} else {
235		✗	if ((num_obus >= 3) && (last_packet_of_frame || (first_elem_with_size <= rem_pkt_size))) {
236			// last fits with forced size, but nothing else
237		✗	num_lebs = write_leb(pkt_ptr, last_elem_size);
238		✗	pkt_ptr += num_lebs;
239		✗	rem_pkt_size -= num_lebs;
240			}
241			// force new packet
242		✗	start_new_packet = 1;
243			}
244
245			// write header and optional extension byte (if not a continued fragment)
246		✗	if (last_obu_hdr >= 0) {
247		✗	*pkt_ptr++ = last_obu_hdr;
248		✗	last_elem_size--;
249		✗	rem_pkt_size--;
250		✗	if (last_obu_ext >= 0) {
251		✗	*pkt_ptr++ = last_obu_ext;
252		✗	last_elem_size--;
253		✗	rem_pkt_size--;
254			}
255			}
256			// copy payload
257		✗	memcpy(pkt_ptr, last_obu_ptr, last_elem_size);
258		✗	pkt_ptr += last_elem_size;
259		✗	rem_pkt_size -= last_elem_size;
260		✗	num_obus++;
261			}
262
263		✗	if (start_new_packet || last_packet_of_frame) {
264		✗	if (num_obus < 4) {
265		✗	aggr_hdr |= num_obus << AV1S_AGGR_HDR_NUM_OBUS;
266			}
267		✗	rtp_ctx->buf[0] = aggr_hdr;
268
269			#ifdef RTPENC_AV1_VERBOSE_TRACE
270			av_log(ctx, AV_LOG_TRACE, "Sending NON-FRAG packet no %d, %ld/%d, %d OBUs (marker=%d)\n",
271			((RTPMuxContext *) ctx->priv_data)->seq,
272			pkt_ptr - rtp_ctx->buf, rtp_ctx->max_payload_size, num_obus, last_packet_of_frame);
273			av_hex_dump_log(ctx, AV_LOG_TRACE, rtp_ctx->buf, FFMIN(pkt_ptr - rtp_ctx->buf, 64));
274			av_log(ctx, AV_LOG_TRACE, "... end at offset %lx:\n", FFMAX((pkt_ptr - rtp_ctx->buf) - 64, 0));
275			av_hex_dump_log(ctx, AV_LOG_TRACE, rtp_ctx->buf + FFMAX((pkt_ptr - rtp_ctx->buf) - 64, 0), FFMIN(pkt_ptr - rtp_ctx->buf, 64));
276			#endif
277
278		✗	ff_rtp_send_data(ctx, rtp_ctx->buf, pkt_ptr - rtp_ctx->buf, last_packet_of_frame);
279
280		✗	rem_pkt_size = rtp_ctx->max_payload_size - 1;
281		✗	pkt_ptr = rtp_ctx->buf + 1;
282		✗	aggr_hdr = 0;
283		✗	num_obus = 0;
284			}
285
286		✗	if (last_packet_of_frame) {
287		✗	break;
288			}
289
290			// check if element needs to be fragmented, otherwise we will deal with
291			// it in the next iteration
292		✗	if ((curr_elem_size > rem_pkt_size) ||
293		✗	((num_obus >= 3) && (curr_elem_size + calc_leb_size(curr_elem_size)) > rem_pkt_size)) {
294		✗	uint32_t frag_size = rem_pkt_size;
295
296			// if there are going more than 3 OBU elements, we are obliged to
297			// have the length field for the last
298		✗	if (num_obus >= 3) {
299			// that's an upper limit of LEBs
300		✗	num_lebs = calc_leb_size(rem_pkt_size - 1);
301		✗	frag_size -= num_lebs;
302
303			// write a fixed number of LEBs, in case the frag_size could
304			// now be specified with one less byte
305		✗	write_leb_n(pkt_ptr, frag_size, num_lebs);
306		✗	pkt_ptr += num_lebs;
307		✗	rem_pkt_size -= num_lebs;
308			}
309
310			// write header and optional extension byte
311		✗	*pkt_ptr++ = curr_obu_hdr;
312		✗	curr_elem_size--;
313		✗	rem_pkt_size--;
314		✗	if (curr_obu_ext >= 0) {
315		✗	*pkt_ptr++ = curr_obu_ext;
316		✗	curr_elem_size--;
317		✗	rem_pkt_size--;
318			}
319
320			// disable header writing for final fragment
321		✗	curr_obu_hdr = -1;
322		✗	curr_obu_ext = -1;
323
324			// send more full packet sized fragments
325			do {
326			// copy payload
327		✗	memcpy(pkt_ptr, curr_obu_ptr, rem_pkt_size);
328		✗	pkt_ptr += rem_pkt_size;
329		✗	curr_obu_ptr += rem_pkt_size;
330		✗	curr_elem_size -= rem_pkt_size;
331		✗	num_obus++;
332
333		✗	aggr_hdr |= AV1F_AGGR_HDR_LAST_FRAG;
334		✗	if (num_obus < 4) {
335		✗	aggr_hdr |= num_obus << AV1S_AGGR_HDR_NUM_OBUS;
336			}
337		✗	rtp_ctx->buf[0] = aggr_hdr;
338
339			#ifdef RTPENC_AV1_VERBOSE_TRACE
340			av_log(ctx, AV_LOG_DEBUG, "Sending FRAG packet no %d, %ld/%d, %d OBUs\n",
341			((RTPMuxContext *) ctx->priv_data)->seq,
342			pkt_ptr - rtp_ctx->buf, rtp_ctx->max_payload_size, num_obus);
343			av_hex_dump_log(ctx, AV_LOG_TRACE, rtp_ctx->buf, FFMIN(pkt_ptr - rtp_ctx->buf, 64));
344			av_log(ctx, AV_LOG_TRACE, "... end at offset %lx:\n", FFMAX((pkt_ptr - rtp_ctx->buf) - 64, 0));
345			av_hex_dump_log(ctx, AV_LOG_TRACE, rtp_ctx->buf + FFMAX((pkt_ptr - rtp_ctx->buf) - 64, 0), FFMIN(pkt_ptr - rtp_ctx->buf, 64));
346			#endif
347
348		✗	ff_rtp_send_data(ctx, rtp_ctx->buf, pkt_ptr - rtp_ctx->buf, 0);
349		✗	rem_pkt_size = rtp_ctx->max_payload_size - 1;
350		✗	pkt_ptr = rtp_ctx->buf + 1;
351
352		✗	aggr_hdr = AV1F_AGGR_HDR_FRAG_CONT;
353		✗	num_obus = 0;
354		✗	} while (curr_elem_size > rem_pkt_size);
355		✗	start_new_packet = 0;
356			}
357
358		✗	if (!frame_size) {
359			// we're done, flush the last packet, set RTP marker bit
360		✗	last_packet_of_frame = 1;
361		✗	goto flush_last_packet;
362			}
363			}
364			}
365