FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* This file is part of FFmpeg.
3			*
4			* FFmpeg is free software; you can redistribute it and/or
5			* modify it under the terms of the GNU Lesser General Public
6			* License as published by the Free Software Foundation; either
7			* version 2.1 of the License, or (at your option) any later version.
8			*
9			* FFmpeg is distributed in the hope that it will be useful,
10			* but WITHOUT ANY WARRANTY; without even the implied warranty of
11			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12			* Lesser General Public License for more details.
13			*
14			* You should have received a copy of the GNU Lesser General Public
15			* License along with FFmpeg; if not, write to the Free Software
16			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17			*/
18
19			#include <inttypes.h>
20			#include <string.h>
21
22			#include "config.h"
23
24			#include "libavutil/avassert.h"
25			#include "libavutil/common.h"
26			#include "libavutil/internal.h"
27			#include "libavutil/log.h"
28			#include "libavutil/mem.h"
29			#include "libavutil/pixdesc.h"
30
31			#include "vaapi_encode.h"
32			#include "encode.h"
33			#include "avcodec.h"
34			#include "refstruct.h"
35
36			const AVCodecHWConfigInternal *const ff_vaapi_encode_hw_configs[] = {
37			HW_CONFIG_ENCODER_FRAMES(VAAPI, VAAPI),
38			NULL,
39			};
40
41			static const char * const picture_type_name[] = { "IDR", "I", "P", "B" };
42
43		✗	static int vaapi_encode_make_packed_header(AVCodecContext *avctx,
44			VAAPIEncodePicture *pic,
45			int type, char *data, size_t bit_len)
46			{
47		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
48			VAStatus vas;
49			VABufferID param_buffer, data_buffer;
50			VABufferID *tmp;
51		✗	VAEncPackedHeaderParameterBuffer params = {
52			.type = type,
53			.bit_length = bit_len,
54			.has_emulation_bytes = 1,
55			};
56
57		✗	tmp = av_realloc_array(pic->param_buffers, sizeof(*tmp), pic->nb_param_buffers + 2);
58		✗	if (!tmp)
59		✗	return AVERROR(ENOMEM);
60		✗	pic->param_buffers = tmp;
61
62		✗	vas = vaCreateBuffer(ctx->hwctx->display, ctx->va_context,
63			VAEncPackedHeaderParameterBufferType,
64			sizeof(params), 1, &params, &param_buffer);
65		✗	if (vas != VA_STATUS_SUCCESS) {
66		✗	av_log(avctx, AV_LOG_ERROR, "Failed to create parameter buffer "
67			"for packed header (type %d): %d (%s).\n",
68			type, vas, vaErrorStr(vas));
69		✗	return AVERROR(EIO);
70			}
71		✗	pic->param_buffers[pic->nb_param_buffers++] = param_buffer;
72
73		✗	vas = vaCreateBuffer(ctx->hwctx->display, ctx->va_context,
74			VAEncPackedHeaderDataBufferType,
75		✗	(bit_len + 7) / 8, 1, data, &data_buffer);
76		✗	if (vas != VA_STATUS_SUCCESS) {
77		✗	av_log(avctx, AV_LOG_ERROR, "Failed to create data buffer "
78			"for packed header (type %d): %d (%s).\n",
79			type, vas, vaErrorStr(vas));
80		✗	return AVERROR(EIO);
81			}
82		✗	pic->param_buffers[pic->nb_param_buffers++] = data_buffer;
83
84		✗	av_log(avctx, AV_LOG_DEBUG, "Packed header buffer (%d) is %#x/%#x "
85			"(%zu bits).\n", type, param_buffer, data_buffer, bit_len);
86		✗	return 0;
87			}
88
89		✗	static int vaapi_encode_make_param_buffer(AVCodecContext *avctx,
90			VAAPIEncodePicture *pic,
91			int type, char *data, size_t len)
92			{
93		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
94			VAStatus vas;
95			VABufferID *tmp;
96			VABufferID buffer;
97
98		✗	tmp = av_realloc_array(pic->param_buffers, sizeof(*tmp), pic->nb_param_buffers + 1);
99		✗	if (!tmp)
100		✗	return AVERROR(ENOMEM);
101		✗	pic->param_buffers = tmp;
102
103		✗	vas = vaCreateBuffer(ctx->hwctx->display, ctx->va_context,
104			type, len, 1, data, &buffer);
105		✗	if (vas != VA_STATUS_SUCCESS) {
106		✗	av_log(avctx, AV_LOG_ERROR, "Failed to create parameter buffer "
107			"(type %d): %d (%s).\n", type, vas, vaErrorStr(vas));
108		✗	return AVERROR(EIO);
109			}
110		✗	pic->param_buffers[pic->nb_param_buffers++] = buffer;
111
112		✗	av_log(avctx, AV_LOG_DEBUG, "Param buffer (%d) is %#x.\n",
113			type, buffer);
114		✗	return 0;
115			}
116
117		✗	static int vaapi_encode_make_misc_param_buffer(AVCodecContext *avctx,
118			VAAPIEncodePicture *pic,
119			int type,
120			const void *data, size_t len)
121			{
122			// Construct the buffer on the stack - 1KB is much larger than any
123			// current misc parameter buffer type (the largest is EncQuality at
124			// 224 bytes).
125			uint8_t buffer[1024];
126		✗	VAEncMiscParameterBuffer header = {
127			.type = type,
128			};
129		✗	size_t buffer_size = sizeof(header) + len;
130		✗	av_assert0(buffer_size <= sizeof(buffer));
131
132		✗	memcpy(buffer, &header, sizeof(header));
133		✗	memcpy(buffer + sizeof(header), data, len);
134
135		✗	return vaapi_encode_make_param_buffer(avctx, pic,
136			VAEncMiscParameterBufferType,
137			buffer, buffer_size);
138			}
139
140		✗	static int vaapi_encode_wait(AVCodecContext *avctx,
141			VAAPIEncodePicture *pic)
142			{
143		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
144			VAStatus vas;
145
146		✗	av_assert0(pic->encode_issued);
147
148		✗	if (pic->encode_complete) {
149			// Already waited for this picture.
150		✗	return 0;
151			}
152
153		✗	av_log(avctx, AV_LOG_DEBUG, "Sync to pic %"PRId64"/%"PRId64" "
154			"(input surface %#x).\n", pic->display_order,
155			pic->encode_order, pic->input_surface);
156
157			#if VA_CHECK_VERSION(1, 9, 0)
158		✗	if (ctx->has_sync_buffer_func) {
159		✗	vas = vaSyncBuffer(ctx->hwctx->display,
160			pic->output_buffer,
161			VA_TIMEOUT_INFINITE);
162		✗	if (vas != VA_STATUS_SUCCESS) {
163		✗	av_log(avctx, AV_LOG_ERROR, "Failed to sync to output buffer completion: "
164			"%d (%s).\n", vas, vaErrorStr(vas));
165		✗	return AVERROR(EIO);
166			}
167			} else
168			#endif
169			{ // If vaSyncBuffer is not implemented, try old version API.
170		✗	vas = vaSyncSurface(ctx->hwctx->display, pic->input_surface);
171		✗	if (vas != VA_STATUS_SUCCESS) {
172		✗	av_log(avctx, AV_LOG_ERROR, "Failed to sync to picture completion: "
173			"%d (%s).\n", vas, vaErrorStr(vas));
174		✗	return AVERROR(EIO);
175			}
176			}
177
178			// Input is definitely finished with now.
179		✗	av_frame_free(&pic->input_image);
180
181		✗	pic->encode_complete = 1;
182		✗	return 0;
183			}
184
185		✗	static int vaapi_encode_make_row_slice(AVCodecContext *avctx,
186			VAAPIEncodePicture *pic)
187			{
188		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
189			VAAPIEncodeSlice *slice;
190			int i, rounding;
191
192		✗	for (i = 0; i < pic->nb_slices; i++)
193		✗	pic->slices[i].row_size = ctx->slice_size;
194
195		✗	rounding = ctx->slice_block_rows - ctx->nb_slices * ctx->slice_size;
196		✗	if (rounding > 0) {
197			// Place rounding error at top and bottom of frame.
198		✗	av_assert0(rounding < pic->nb_slices);
199			// Some Intel drivers contain a bug where the encoder will fail
200			// if the last slice is smaller than the one before it. Since
201			// that's straightforward to avoid here, just do so.
202		✗	if (rounding <= 2) {
203		✗	for (i = 0; i < rounding; i++)
204		✗	++pic->slices[i].row_size;
205			} else {
206		✗	for (i = 0; i < (rounding + 1) / 2; i++)
207		✗	++pic->slices[pic->nb_slices - i - 1].row_size;
208		✗	for (i = 0; i < rounding / 2; i++)
209		✗	++pic->slices[i].row_size;
210			}
211		✗	} else if (rounding < 0) {
212			// Remove rounding error from last slice only.
213		✗	av_assert0(rounding < ctx->slice_size);
214		✗	pic->slices[pic->nb_slices - 1].row_size += rounding;
215			}
216
217		✗	for (i = 0; i < pic->nb_slices; i++) {
218		✗	slice = &pic->slices[i];
219		✗	slice->index = i;
220		✗	if (i == 0) {
221		✗	slice->row_start = 0;
222		✗	slice->block_start = 0;
223			} else {
224		✗	const VAAPIEncodeSlice *prev = &pic->slices[i - 1];
225		✗	slice->row_start = prev->row_start + prev->row_size;
226		✗	slice->block_start = prev->block_start + prev->block_size;
227			}
228		✗	slice->block_size = slice->row_size * ctx->slice_block_cols;
229
230		✗	av_log(avctx, AV_LOG_DEBUG, "Slice %d: %d-%d (%d rows), "
231			"%d-%d (%d blocks).\n", i, slice->row_start,
232		✗	slice->row_start + slice->row_size - 1, slice->row_size,
233		✗	slice->block_start, slice->block_start + slice->block_size - 1,
234			slice->block_size);
235			}
236
237		✗	return 0;
238			}
239
240		✗	static int vaapi_encode_make_tile_slice(AVCodecContext *avctx,
241			VAAPIEncodePicture *pic)
242			{
243		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
244			VAAPIEncodeSlice *slice;
245			int i, j, index;
246
247		✗	for (i = 0; i < ctx->tile_cols; i++) {
248		✗	for (j = 0; j < ctx->tile_rows; j++) {
249		✗	index = j * ctx->tile_cols + i;
250		✗	slice = &pic->slices[index];
251		✗	slice->index = index;
252
253		✗	pic->slices[index].block_start = ctx->col_bd[i] +
254		✗	ctx->row_bd[j] * ctx->slice_block_cols;
255		✗	pic->slices[index].block_size = ctx->row_height[j] * ctx->col_width[i];
256
257		✗	av_log(avctx, AV_LOG_DEBUG, "Slice %2d: (%2d, %2d) start at: %4d "
258			"width:%2d height:%2d (%d blocks).\n", index, ctx->col_bd[i],
259			ctx->row_bd[j], slice->block_start, ctx->col_width[i],
260			ctx->row_height[j], slice->block_size);
261			}
262			}
263
264		✗	return 0;
265			}
266
267		✗	static int vaapi_encode_issue(AVCodecContext *avctx,
268			VAAPIEncodePicture *pic)
269			{
270		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
271			VAAPIEncodeSlice *slice;
272			VAStatus vas;
273			int err, i;
274			char data[MAX_PARAM_BUFFER_SIZE];
275			size_t bit_len;
276			av_unused AVFrameSideData *sd;
277
278		✗	av_log(avctx, AV_LOG_DEBUG, "Issuing encode for pic %"PRId64"/%"PRId64" "
279			"as type %s.\n", pic->display_order, pic->encode_order,
280		✗	picture_type_name[pic->type]);
281		✗	if (pic->nb_refs[0] == 0 && pic->nb_refs[1] == 0) {
282		✗	av_log(avctx, AV_LOG_DEBUG, "No reference pictures.\n");
283			} else {
284		✗	av_log(avctx, AV_LOG_DEBUG, "L0 refers to");
285		✗	for (i = 0; i < pic->nb_refs[0]; i++) {
286		✗	av_log(avctx, AV_LOG_DEBUG, " %"PRId64"/%"PRId64,
287		✗	pic->refs[0][i]->display_order, pic->refs[0][i]->encode_order);
288			}
289		✗	av_log(avctx, AV_LOG_DEBUG, ".\n");
290
291		✗	if (pic->nb_refs[1]) {
292		✗	av_log(avctx, AV_LOG_DEBUG, "L1 refers to");
293		✗	for (i = 0; i < pic->nb_refs[1]; i++) {
294		✗	av_log(avctx, AV_LOG_DEBUG, " %"PRId64"/%"PRId64,
295		✗	pic->refs[1][i]->display_order, pic->refs[1][i]->encode_order);
296			}
297		✗	av_log(avctx, AV_LOG_DEBUG, ".\n");
298			}
299			}
300
301		✗	av_assert0(!pic->encode_issued);
302		✗	for (i = 0; i < pic->nb_refs[0]; i++) {
303		✗	av_assert0(pic->refs[0][i]);
304		✗	av_assert0(pic->refs[0][i]->encode_issued);
305			}
306		✗	for (i = 0; i < pic->nb_refs[1]; i++) {
307		✗	av_assert0(pic->refs[1][i]);
308		✗	av_assert0(pic->refs[1][i]->encode_issued);
309			}
310
311		✗	av_log(avctx, AV_LOG_DEBUG, "Input surface is %#x.\n", pic->input_surface);
312
313		✗	pic->recon_image = av_frame_alloc();
314		✗	if (!pic->recon_image) {
315		✗	err = AVERROR(ENOMEM);
316		✗	goto fail;
317			}
318
319		✗	err = av_hwframe_get_buffer(ctx->recon_frames_ref, pic->recon_image, 0);
320		✗	if (err < 0) {
321		✗	err = AVERROR(ENOMEM);
322		✗	goto fail;
323			}
324		✗	pic->recon_surface = (VASurfaceID)(uintptr_t)pic->recon_image->data[3];
325		✗	av_log(avctx, AV_LOG_DEBUG, "Recon surface is %#x.\n", pic->recon_surface);
326
327		✗	pic->output_buffer_ref = ff_refstruct_pool_get(ctx->output_buffer_pool);
328		✗	if (!pic->output_buffer_ref) {
329		✗	err = AVERROR(ENOMEM);
330		✗	goto fail;
331			}
332		✗	pic->output_buffer = *pic->output_buffer_ref;
333		✗	av_log(avctx, AV_LOG_DEBUG, "Output buffer is %#x.\n",
334			pic->output_buffer);
335
336		✗	if (ctx->codec->picture_params_size > 0) {
337		✗	pic->codec_picture_params = av_malloc(ctx->codec->picture_params_size);
338		✗	if (!pic->codec_picture_params)
339		✗	goto fail;
340		✗	memcpy(pic->codec_picture_params, ctx->codec_picture_params,
341		✗	ctx->codec->picture_params_size);
342			} else {
343		✗	av_assert0(!ctx->codec_picture_params);
344			}
345
346		✗	pic->nb_param_buffers = 0;
347
348		✗	if (pic->type == PICTURE_TYPE_IDR && ctx->codec->init_sequence_params) {
349		✗	err = vaapi_encode_make_param_buffer(avctx, pic,
350			VAEncSequenceParameterBufferType,
351		✗	ctx->codec_sequence_params,
352		✗	ctx->codec->sequence_params_size);
353		✗	if (err < 0)
354		✗	goto fail;
355			}
356
357		✗	if (pic->type == PICTURE_TYPE_IDR) {
358		✗	for (i = 0; i < ctx->nb_global_params; i++) {
359		✗	err = vaapi_encode_make_misc_param_buffer(avctx, pic,
360			ctx->global_params_type[i],
361			ctx->global_params[i],
362			ctx->global_params_size[i]);
363		✗	if (err < 0)
364		✗	goto fail;
365			}
366			}
367
368		✗	if (ctx->codec->init_picture_params) {
369		✗	err = ctx->codec->init_picture_params(avctx, pic);
370		✗	if (err < 0) {
371		✗	av_log(avctx, AV_LOG_ERROR, "Failed to initialise picture "
372			"parameters: %d.\n", err);
373		✗	goto fail;
374			}
375		✗	err = vaapi_encode_make_param_buffer(avctx, pic,
376			VAEncPictureParameterBufferType,
377		✗	pic->codec_picture_params,
378		✗	ctx->codec->picture_params_size);
379		✗	if (err < 0)
380		✗	goto fail;
381			}
382
383			#if VA_CHECK_VERSION(1, 5, 0)
384		✗	if (ctx->max_frame_size) {
385		✗	err = vaapi_encode_make_misc_param_buffer(avctx, pic,
386			VAEncMiscParameterTypeMaxFrameSize,
387		✗	&ctx->mfs_params,
388			sizeof(ctx->mfs_params));
389		✗	if (err < 0)
390		✗	goto fail;
391			}
392			#endif
393
394		✗	if (pic->type == PICTURE_TYPE_IDR) {
395		✗	if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_SEQUENCE &&
396		✗	ctx->codec->write_sequence_header) {
397		✗	bit_len = 8 * sizeof(data);
398		✗	err = ctx->codec->write_sequence_header(avctx, data, &bit_len);
399		✗	if (err < 0) {
400		✗	av_log(avctx, AV_LOG_ERROR, "Failed to write per-sequence "
401			"header: %d.\n", err);
402		✗	goto fail;
403			}
404		✗	err = vaapi_encode_make_packed_header(avctx, pic,
405		✗	ctx->codec->sequence_header_type,
406			data, bit_len);
407		✗	if (err < 0)
408		✗	goto fail;
409			}
410			}
411
412		✗	if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_PICTURE &&
413		✗	ctx->codec->write_picture_header) {
414		✗	bit_len = 8 * sizeof(data);
415		✗	err = ctx->codec->write_picture_header(avctx, pic, data, &bit_len);
416		✗	if (err < 0) {
417		✗	av_log(avctx, AV_LOG_ERROR, "Failed to write per-picture "
418			"header: %d.\n", err);
419		✗	goto fail;
420			}
421		✗	err = vaapi_encode_make_packed_header(avctx, pic,
422		✗	ctx->codec->picture_header_type,
423			data, bit_len);
424		✗	if (err < 0)
425		✗	goto fail;
426			}
427
428		✗	if (ctx->codec->write_extra_buffer) {
429		✗	for (i = 0;; i++) {
430		✗	size_t len = sizeof(data);
431			int type;
432		✗	err = ctx->codec->write_extra_buffer(avctx, pic, i, &type,
433			data, &len);
434		✗	if (err == AVERROR_EOF)
435		✗	break;
436		✗	if (err < 0) {
437		✗	av_log(avctx, AV_LOG_ERROR, "Failed to write extra "
438			"buffer %d: %d.\n", i, err);
439		✗	goto fail;
440			}
441
442		✗	err = vaapi_encode_make_param_buffer(avctx, pic, type,
443			data, len);
444		✗	if (err < 0)
445		✗	goto fail;
446			}
447			}
448
449		✗	if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_MISC &&
450		✗	ctx->codec->write_extra_header) {
451		✗	for (i = 0;; i++) {
452			int type;
453		✗	bit_len = 8 * sizeof(data);
454		✗	err = ctx->codec->write_extra_header(avctx, pic, i, &type,
455			data, &bit_len);
456		✗	if (err == AVERROR_EOF)
457		✗	break;
458		✗	if (err < 0) {
459		✗	av_log(avctx, AV_LOG_ERROR, "Failed to write extra "
460			"header %d: %d.\n", i, err);
461		✗	goto fail;
462			}
463
464		✗	err = vaapi_encode_make_packed_header(avctx, pic, type,
465			data, bit_len);
466		✗	if (err < 0)
467		✗	goto fail;
468			}
469			}
470
471		✗	if (pic->nb_slices == 0)
472		✗	pic->nb_slices = ctx->nb_slices;
473		✗	if (pic->nb_slices > 0) {
474		✗	pic->slices = av_calloc(pic->nb_slices, sizeof(*pic->slices));
475		✗	if (!pic->slices) {
476		✗	err = AVERROR(ENOMEM);
477		✗	goto fail;
478			}
479
480		✗	if (ctx->tile_rows && ctx->tile_cols)
481		✗	vaapi_encode_make_tile_slice(avctx, pic);
482			else
483		✗	vaapi_encode_make_row_slice(avctx, pic);
484			}
485
486		✗	for (i = 0; i < pic->nb_slices; i++) {
487		✗	slice = &pic->slices[i];
488
489		✗	if (ctx->codec->slice_params_size > 0) {
490		✗	slice->codec_slice_params = av_mallocz(ctx->codec->slice_params_size);
491		✗	if (!slice->codec_slice_params) {
492		✗	err = AVERROR(ENOMEM);
493		✗	goto fail;
494			}
495			}
496
497		✗	if (ctx->codec->init_slice_params) {
498		✗	err = ctx->codec->init_slice_params(avctx, pic, slice);
499		✗	if (err < 0) {
500		✗	av_log(avctx, AV_LOG_ERROR, "Failed to initialise slice "
501			"parameters: %d.\n", err);
502		✗	goto fail;
503			}
504			}
505
506		✗	if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_SLICE &&
507		✗	ctx->codec->write_slice_header) {
508		✗	bit_len = 8 * sizeof(data);
509		✗	err = ctx->codec->write_slice_header(avctx, pic, slice,
510			data, &bit_len);
511		✗	if (err < 0) {
512		✗	av_log(avctx, AV_LOG_ERROR, "Failed to write per-slice "
513			"header: %d.\n", err);
514		✗	goto fail;
515			}
516		✗	err = vaapi_encode_make_packed_header(avctx, pic,
517		✗	ctx->codec->slice_header_type,
518			data, bit_len);
519		✗	if (err < 0)
520		✗	goto fail;
521			}
522
523		✗	if (ctx->codec->init_slice_params) {
524		✗	err = vaapi_encode_make_param_buffer(avctx, pic,
525			VAEncSliceParameterBufferType,
526		✗	slice->codec_slice_params,
527		✗	ctx->codec->slice_params_size);
528		✗	if (err < 0)
529		✗	goto fail;
530			}
531			}
532
533			#if VA_CHECK_VERSION(1, 0, 0)
534		✗	sd = av_frame_get_side_data(pic->input_image,
535			AV_FRAME_DATA_REGIONS_OF_INTEREST);
536		✗	if (sd && ctx->roi_allowed) {
537			const AVRegionOfInterest *roi;
538			uint32_t roi_size;
539			VAEncMiscParameterBufferROI param_roi;
540			int nb_roi, i, v;
541
542		✗	roi = (const AVRegionOfInterest*)sd->data;
543		✗	roi_size = roi->self_size;
544		✗	av_assert0(roi_size && sd->size % roi_size == 0);
545		✗	nb_roi = sd->size / roi_size;
546		✗	if (nb_roi > ctx->roi_max_regions) {
547		✗	if (!ctx->roi_warned) {
548		✗	av_log(avctx, AV_LOG_WARNING, "More ROIs set than "
549			"supported by driver (%d > %d).\n",
550			nb_roi, ctx->roi_max_regions);
551		✗	ctx->roi_warned = 1;
552			}
553		✗	nb_roi = ctx->roi_max_regions;
554			}
555
556		✗	pic->roi = av_calloc(nb_roi, sizeof(*pic->roi));
557		✗	if (!pic->roi) {
558		✗	err = AVERROR(ENOMEM);
559		✗	goto fail;
560			}
561			// For overlapping regions, the first in the array takes priority.
562		✗	for (i = 0; i < nb_roi; i++) {
563		✗	roi = (const AVRegionOfInterest*)(sd->data + roi_size * i);
564
565		✗	av_assert0(roi->qoffset.den != 0);
566		✗	v = roi->qoffset.num * ctx->roi_quant_range / roi->qoffset.den;
567		✗	av_log(avctx, AV_LOG_DEBUG, "ROI: (%d,%d)-(%d,%d) -> %+d.\n",
568		✗	roi->top, roi->left, roi->bottom, roi->right, v);
569
570		✗	pic->roi[i] = (VAEncROI) {
571			.roi_rectangle = {
572		✗	.x = roi->left,
573		✗	.y = roi->top,
574		✗	.width = roi->right - roi->left,
575		✗	.height = roi->bottom - roi->top,
576			},
577		✗	.roi_value = av_clip_int8(v),
578			};
579			}
580
581		✗	param_roi = (VAEncMiscParameterBufferROI) {
582			.num_roi = nb_roi,
583			.max_delta_qp = INT8_MAX,
584			.min_delta_qp = INT8_MIN,
585		✗	.roi = pic->roi,
586			.roi_flags.bits.roi_value_is_qp_delta = 1,
587			};
588
589		✗	err = vaapi_encode_make_misc_param_buffer(avctx, pic,
590			VAEncMiscParameterTypeROI,
591			&param_roi,
592			sizeof(param_roi));
593		✗	if (err < 0)
594		✗	goto fail;
595			}
596			#endif
597
598		✗	vas = vaBeginPicture(ctx->hwctx->display, ctx->va_context,
599			pic->input_surface);
600		✗	if (vas != VA_STATUS_SUCCESS) {
601		✗	av_log(avctx, AV_LOG_ERROR, "Failed to begin picture encode issue: "
602			"%d (%s).\n", vas, vaErrorStr(vas));
603		✗	err = AVERROR(EIO);
604		✗	goto fail_with_picture;
605			}
606
607		✗	vas = vaRenderPicture(ctx->hwctx->display, ctx->va_context,
608			pic->param_buffers, pic->nb_param_buffers);
609		✗	if (vas != VA_STATUS_SUCCESS) {
610		✗	av_log(avctx, AV_LOG_ERROR, "Failed to upload encode parameters: "
611			"%d (%s).\n", vas, vaErrorStr(vas));
612		✗	err = AVERROR(EIO);
613		✗	goto fail_with_picture;
614			}
615
616		✗	vas = vaEndPicture(ctx->hwctx->display, ctx->va_context);
617		✗	if (vas != VA_STATUS_SUCCESS) {
618		✗	av_log(avctx, AV_LOG_ERROR, "Failed to end picture encode issue: "
619			"%d (%s).\n", vas, vaErrorStr(vas));
620		✗	err = AVERROR(EIO);
621			// vaRenderPicture() has been called here, so we should not destroy
622			// the parameter buffers unless separate destruction is required.
623			if (CONFIG_VAAPI_1 || ctx->hwctx->driver_quirks &
624			AV_VAAPI_DRIVER_QUIRK_RENDER_PARAM_BUFFERS)
625		✗	goto fail;
626			else
627			goto fail_at_end;
628			}
629
630			if (CONFIG_VAAPI_1 || ctx->hwctx->driver_quirks &
631			AV_VAAPI_DRIVER_QUIRK_RENDER_PARAM_BUFFERS) {
632		✗	for (i = 0; i < pic->nb_param_buffers; i++) {
633		✗	vas = vaDestroyBuffer(ctx->hwctx->display,
634		✗	pic->param_buffers[i]);
635		✗	if (vas != VA_STATUS_SUCCESS) {
636		✗	av_log(avctx, AV_LOG_ERROR, "Failed to destroy "
637			"param buffer %#x: %d (%s).\n",
638		✗	pic->param_buffers[i], vas, vaErrorStr(vas));
639			// And ignore.
640			}
641			}
642			}
643
644		✗	pic->encode_issued = 1;
645
646		✗	return 0;
647
648		✗	fail_with_picture:
649		✗	vaEndPicture(ctx->hwctx->display, ctx->va_context);
650		✗	fail:
651		✗	for(i = 0; i < pic->nb_param_buffers; i++)
652		✗	vaDestroyBuffer(ctx->hwctx->display, pic->param_buffers[i]);
653		✗	if (pic->slices) {
654		✗	for (i = 0; i < pic->nb_slices; i++)
655		✗	av_freep(&pic->slices[i].codec_slice_params);
656			}
657		✗	fail_at_end:
658		✗	av_freep(&pic->codec_picture_params);
659		✗	av_freep(&pic->param_buffers);
660		✗	av_freep(&pic->slices);
661		✗	av_freep(&pic->roi);
662		✗	av_frame_free(&pic->recon_image);
663		✗	ff_refstruct_unref(&pic->output_buffer_ref);
664		✗	pic->output_buffer = VA_INVALID_ID;
665		✗	return err;
666			}
667
668		✗	static int vaapi_encode_set_output_property(AVCodecContext *avctx,
669			VAAPIEncodePicture *pic,
670			AVPacket *pkt)
671			{
672		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
673
674		✗	if (pic->type == PICTURE_TYPE_IDR)
675		✗	pkt->flags |= AV_PKT_FLAG_KEY;
676
677		✗	pkt->pts = pic->pts;
678		✗	pkt->duration = pic->duration;
679
680			// for no-delay encoders this is handled in generic codec
681		✗	if (avctx->codec->capabilities & AV_CODEC_CAP_DELAY &&
682		✗	avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) {
683		✗	pkt->opaque = pic->opaque;
684		✗	pkt->opaque_ref = pic->opaque_ref;
685		✗	pic->opaque_ref = NULL;
686			}
687
688		✗	if (ctx->codec->flags & FLAG_TIMESTAMP_NO_DELAY) {
689		✗	pkt->dts = pkt->pts;
690		✗	return 0;
691			}
692
693		✗	if (ctx->output_delay == 0) {
694		✗	pkt->dts = pkt->pts;
695		✗	} else if (pic->encode_order < ctx->decode_delay) {
696		✗	if (ctx->ts_ring[pic->encode_order] < INT64_MIN + ctx->dts_pts_diff)
697		✗	pkt->dts = INT64_MIN;
698			else
699		✗	pkt->dts = ctx->ts_ring[pic->encode_order] - ctx->dts_pts_diff;
700			} else {
701		✗	pkt->dts = ctx->ts_ring[(pic->encode_order - ctx->decode_delay) %
702		✗	(3 * ctx->output_delay + ctx->async_depth)];
703			}
704
705		✗	return 0;
706			}
707
708		✗	static int vaapi_encode_get_coded_buffer_size(AVCodecContext *avctx, VABufferID buf_id)
709			{
710		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
711			VACodedBufferSegment *buf_list, *buf;
712		✗	int size = 0;
713			VAStatus vas;
714			int err;
715
716		✗	vas = vaMapBuffer(ctx->hwctx->display, buf_id,
717			(void**)&buf_list);
718		✗	if (vas != VA_STATUS_SUCCESS) {
719		✗	av_log(avctx, AV_LOG_ERROR, "Failed to map output buffers: "
720			"%d (%s).\n", vas, vaErrorStr(vas));
721		✗	err = AVERROR(EIO);
722		✗	return err;
723			}
724
725		✗	for (buf = buf_list; buf; buf = buf->next)
726		✗	size += buf->size;
727
728		✗	vas = vaUnmapBuffer(ctx->hwctx->display, buf_id);
729		✗	if (vas != VA_STATUS_SUCCESS) {
730		✗	av_log(avctx, AV_LOG_ERROR, "Failed to unmap output buffers: "
731			"%d (%s).\n", vas, vaErrorStr(vas));
732		✗	err = AVERROR(EIO);
733		✗	return err;
734			}
735
736		✗	return size;
737			}
738
739		✗	static int vaapi_encode_get_coded_buffer_data(AVCodecContext *avctx,
740			VABufferID buf_id, uint8_t **dst)
741			{
742		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
743			VACodedBufferSegment *buf_list, *buf;
744			VAStatus vas;
745			int err;
746
747		✗	vas = vaMapBuffer(ctx->hwctx->display, buf_id,
748			(void**)&buf_list);
749		✗	if (vas != VA_STATUS_SUCCESS) {
750		✗	av_log(avctx, AV_LOG_ERROR, "Failed to map output buffers: "
751			"%d (%s).\n", vas, vaErrorStr(vas));
752		✗	err = AVERROR(EIO);
753		✗	return err;
754			}
755
756		✗	for (buf = buf_list; buf; buf = buf->next) {
757		✗	av_log(avctx, AV_LOG_DEBUG, "Output buffer: %u bytes "
758			"(status %08x).\n", buf->size, buf->status);
759
760		✗	memcpy(*dst, buf->buf, buf->size);
761		✗	*dst += buf->size;
762			}
763
764		✗	vas = vaUnmapBuffer(ctx->hwctx->display, buf_id);
765		✗	if (vas != VA_STATUS_SUCCESS) {
766		✗	av_log(avctx, AV_LOG_ERROR, "Failed to unmap output buffers: "
767			"%d (%s).\n", vas, vaErrorStr(vas));
768		✗	err = AVERROR(EIO);
769		✗	return err;
770			}
771
772		✗	return 0;
773			}
774
775		✗	static int vaapi_encode_get_coded_data(AVCodecContext *avctx,
776			VAAPIEncodePicture *pic, AVPacket *pkt)
777			{
778		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
779			VABufferID output_buffer_prev;
780		✗	int total_size = 0;
781			uint8_t *ptr;
782			int ret;
783
784		✗	if (ctx->coded_buffer_ref) {
785		✗	output_buffer_prev = *ctx->coded_buffer_ref;
786		✗	ret = vaapi_encode_get_coded_buffer_size(avctx, output_buffer_prev);
787		✗	if (ret < 0)
788		✗	goto end;
789		✗	total_size += ret;
790			}
791
792		✗	ret = vaapi_encode_get_coded_buffer_size(avctx, pic->output_buffer);
793		✗	if (ret < 0)
794		✗	goto end;
795		✗	total_size += ret;
796
797		✗	ret = ff_get_encode_buffer(avctx, pkt, total_size, 0);
798		✗	if (ret < 0)
799		✗	goto end;
800		✗	ptr = pkt->data;
801
802		✗	if (ctx->coded_buffer_ref) {
803		✗	ret = vaapi_encode_get_coded_buffer_data(avctx, output_buffer_prev, &ptr);
804		✗	if (ret < 0)
805		✗	goto end;
806			}
807
808		✗	ret = vaapi_encode_get_coded_buffer_data(avctx, pic->output_buffer, &ptr);
809		✗	if (ret < 0)
810		✗	goto end;
811
812		✗	end:
813		✗	ff_refstruct_unref(&ctx->coded_buffer_ref);
814		✗	ff_refstruct_unref(&pic->output_buffer_ref);
815		✗	pic->output_buffer = VA_INVALID_ID;
816
817		✗	return ret;
818			}
819
820		✗	static int vaapi_encode_output(AVCodecContext *avctx,
821			VAAPIEncodePicture *pic, AVPacket *pkt)
822			{
823		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
824		✗	AVPacket *pkt_ptr = pkt;
825			int err;
826
827		✗	err = vaapi_encode_wait(avctx, pic);
828		✗	if (err < 0)
829		✗	return err;
830
831		✗	if (pic->non_independent_frame) {
832		✗	av_assert0(!ctx->coded_buffer_ref);
833		✗	ctx->coded_buffer_ref = ff_refstruct_ref(pic->output_buffer_ref);
834
835		✗	if (pic->tail_size) {
836		✗	if (ctx->tail_pkt->size) {
837		✗	err = AVERROR_BUG;
838		✗	goto end;
839			}
840
841		✗	err = ff_get_encode_buffer(avctx, ctx->tail_pkt, pic->tail_size, 0);
842		✗	if (err < 0)
843		✗	goto end;
844
845		✗	memcpy(ctx->tail_pkt->data, pic->tail_data, pic->tail_size);
846		✗	pkt_ptr = ctx->tail_pkt;
847			}
848			} else {
849		✗	err = vaapi_encode_get_coded_data(avctx, pic, pkt);
850		✗	if (err < 0)
851		✗	goto end;
852			}
853
854		✗	av_log(avctx, AV_LOG_DEBUG, "Output read for pic %"PRId64"/%"PRId64".\n",
855			pic->display_order, pic->encode_order);
856
857		✗	vaapi_encode_set_output_property(avctx, pic, pkt_ptr);
858
859		✗	end:
860		✗	ff_refstruct_unref(&pic->output_buffer_ref);
861		✗	pic->output_buffer = VA_INVALID_ID;
862		✗	return err;
863			}
864
865		✗	static int vaapi_encode_discard(AVCodecContext *avctx,
866			VAAPIEncodePicture *pic)
867			{
868		✗	vaapi_encode_wait(avctx, pic);
869
870		✗	if (pic->output_buffer_ref) {
871		✗	av_log(avctx, AV_LOG_DEBUG, "Discard output for pic "
872			"%"PRId64"/%"PRId64".\n",
873			pic->display_order, pic->encode_order);
874
875		✗	ff_refstruct_unref(&pic->output_buffer_ref);
876		✗	pic->output_buffer = VA_INVALID_ID;
877			}
878
879		✗	return 0;
880			}
881
882		✗	static VAAPIEncodePicture *vaapi_encode_alloc(AVCodecContext *avctx)
883			{
884		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
885			VAAPIEncodePicture *pic;
886
887		✗	pic = av_mallocz(sizeof(*pic));
888		✗	if (!pic)
889		✗	return NULL;
890
891		✗	if (ctx->codec->picture_priv_data_size > 0) {
892		✗	pic->priv_data = av_mallocz(ctx->codec->picture_priv_data_size);
893		✗	if (!pic->priv_data) {
894		✗	av_freep(&pic);
895		✗	return NULL;
896			}
897			}
898
899		✗	pic->input_surface = VA_INVALID_ID;
900		✗	pic->recon_surface = VA_INVALID_ID;
901		✗	pic->output_buffer = VA_INVALID_ID;
902
903		✗	return pic;
904			}
905
906		✗	static int vaapi_encode_free(AVCodecContext *avctx,
907			VAAPIEncodePicture *pic)
908			{
909			int i;
910
911		✗	if (pic->encode_issued)
912		✗	vaapi_encode_discard(avctx, pic);
913
914		✗	if (pic->slices) {
915		✗	for (i = 0; i < pic->nb_slices; i++)
916		✗	av_freep(&pic->slices[i].codec_slice_params);
917			}
918
919		✗	av_frame_free(&pic->input_image);
920		✗	av_frame_free(&pic->recon_image);
921
922		✗	av_buffer_unref(&pic->opaque_ref);
923
924		✗	av_freep(&pic->param_buffers);
925		✗	av_freep(&pic->slices);
926			// Output buffer should already be destroyed.
927		✗	av_assert0(pic->output_buffer == VA_INVALID_ID);
928
929		✗	av_freep(&pic->priv_data);
930		✗	av_freep(&pic->codec_picture_params);
931		✗	av_freep(&pic->roi);
932
933		✗	av_free(pic);
934
935		✗	return 0;
936			}
937
938		✗	static void vaapi_encode_add_ref(AVCodecContext *avctx,
939			VAAPIEncodePicture *pic,
940			VAAPIEncodePicture *target,
941			int is_ref, int in_dpb, int prev)
942			{
943		✗	int refs = 0;
944
945		✗	if (is_ref) {
946		✗	av_assert0(pic != target);
947		✗	av_assert0(pic->nb_refs[0] < MAX_PICTURE_REFERENCES &&
948			pic->nb_refs[1] < MAX_PICTURE_REFERENCES);
949		✗	if (target->display_order < pic->display_order)
950		✗	pic->refs[0][pic->nb_refs[0]++] = target;
951			else
952		✗	pic->refs[1][pic->nb_refs[1]++] = target;
953		✗	++refs;
954			}
955
956		✗	if (in_dpb) {
957		✗	av_assert0(pic->nb_dpb_pics < MAX_DPB_SIZE);
958		✗	pic->dpb[pic->nb_dpb_pics++] = target;
959		✗	++refs;
960			}
961
962		✗	if (prev) {
963		✗	av_assert0(!pic->prev);
964		✗	pic->prev = target;
965		✗	++refs;
966			}
967
968		✗	target->ref_count[0] += refs;
969		✗	target->ref_count[1] += refs;
970		✗	}
971
972		✗	static void vaapi_encode_remove_refs(AVCodecContext *avctx,
973			VAAPIEncodePicture *pic,
974			int level)
975			{
976			int i;
977
978		✗	if (pic->ref_removed[level])
979		✗	return;
980
981		✗	for (i = 0; i < pic->nb_refs[0]; i++) {
982		✗	av_assert0(pic->refs[0][i]);
983		✗	--pic->refs[0][i]->ref_count[level];
984		✗	av_assert0(pic->refs[0][i]->ref_count[level] >= 0);
985			}
986
987		✗	for (i = 0; i < pic->nb_refs[1]; i++) {
988		✗	av_assert0(pic->refs[1][i]);
989		✗	--pic->refs[1][i]->ref_count[level];
990		✗	av_assert0(pic->refs[1][i]->ref_count[level] >= 0);
991			}
992
993		✗	for (i = 0; i < pic->nb_dpb_pics; i++) {
994		✗	av_assert0(pic->dpb[i]);
995		✗	--pic->dpb[i]->ref_count[level];
996		✗	av_assert0(pic->dpb[i]->ref_count[level] >= 0);
997			}
998
999		✗	av_assert0(pic->prev || pic->type == PICTURE_TYPE_IDR);
1000		✗	if (pic->prev) {
1001		✗	--pic->prev->ref_count[level];
1002		✗	av_assert0(pic->prev->ref_count[level] >= 0);
1003			}
1004
1005		✗	pic->ref_removed[level] = 1;
1006			}
1007
1008		✗	static void vaapi_encode_set_b_pictures(AVCodecContext *avctx,
1009			VAAPIEncodePicture *start,
1010			VAAPIEncodePicture *end,
1011			VAAPIEncodePicture *prev,
1012			int current_depth,
1013			VAAPIEncodePicture **last)
1014			{
1015		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
1016			VAAPIEncodePicture *pic, *next, *ref;
1017			int i, len;
1018
1019		✗	av_assert0(start && end && start != end && start->next != end);
1020
1021			// If we are at the maximum depth then encode all pictures as
1022			// non-referenced B-pictures. Also do this if there is exactly one
1023			// picture left, since there will be nothing to reference it.
1024		✗	if (current_depth == ctx->max_b_depth || start->next->next == end) {
1025		✗	for (pic = start->next; pic; pic = pic->next) {
1026		✗	if (pic == end)
1027		✗	break;
1028		✗	pic->type = PICTURE_TYPE_B;
1029		✗	pic->b_depth = current_depth;
1030
1031		✗	vaapi_encode_add_ref(avctx, pic, start, 1, 1, 0);
1032		✗	vaapi_encode_add_ref(avctx, pic, end, 1, 1, 0);
1033		✗	vaapi_encode_add_ref(avctx, pic, prev, 0, 0, 1);
1034
1035		✗	for (ref = end->refs[1][0]; ref; ref = ref->refs[1][0])
1036		✗	vaapi_encode_add_ref(avctx, pic, ref, 0, 1, 0);
1037			}
1038		✗	*last = prev;
1039
1040			} else {
1041			// Split the current list at the midpoint with a referenced
1042			// B-picture, then descend into each side separately.
1043		✗	len = 0;
1044		✗	for (pic = start->next; pic != end; pic = pic->next)
1045		✗	++len;
1046		✗	for (pic = start->next, i = 1; 2 * i < len; pic = pic->next, i++);
1047
1048		✗	pic->type = PICTURE_TYPE_B;
1049		✗	pic->b_depth = current_depth;
1050
1051		✗	pic->is_reference = 1;
1052
1053		✗	vaapi_encode_add_ref(avctx, pic, pic, 0, 1, 0);
1054		✗	vaapi_encode_add_ref(avctx, pic, start, 1, 1, 0);
1055		✗	vaapi_encode_add_ref(avctx, pic, end, 1, 1, 0);
1056		✗	vaapi_encode_add_ref(avctx, pic, prev, 0, 0, 1);
1057
1058		✗	for (ref = end->refs[1][0]; ref; ref = ref->refs[1][0])
1059		✗	vaapi_encode_add_ref(avctx, pic, ref, 0, 1, 0);
1060
1061		✗	if (i > 1)
1062		✗	vaapi_encode_set_b_pictures(avctx, start, pic, pic,
1063			current_depth + 1, &next);
1064			else
1065		✗	next = pic;
1066
1067		✗	vaapi_encode_set_b_pictures(avctx, pic, end, next,
1068			current_depth + 1, last);
1069			}
1070		✗	}
1071
1072		✗	static void vaapi_encode_add_next_prev(AVCodecContext *avctx,
1073			VAAPIEncodePicture *pic)
1074			{
1075		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
1076			int i;
1077
1078		✗	if (!pic)
1079		✗	return;
1080
1081		✗	if (pic->type == PICTURE_TYPE_IDR) {
1082		✗	for (i = 0; i < ctx->nb_next_prev; i++) {
1083		✗	--ctx->next_prev[i]->ref_count[0];
1084		✗	ctx->next_prev[i] = NULL;
1085			}
1086		✗	ctx->next_prev[0] = pic;
1087		✗	++pic->ref_count[0];
1088		✗	ctx->nb_next_prev = 1;
1089
1090		✗	return;
1091			}
1092
1093		✗	if (ctx->nb_next_prev < MAX_PICTURE_REFERENCES) {
1094		✗	ctx->next_prev[ctx->nb_next_prev++] = pic;
1095		✗	++pic->ref_count[0];
1096			} else {
1097		✗	--ctx->next_prev[0]->ref_count[0];
1098		✗	for (i = 0; i < MAX_PICTURE_REFERENCES - 1; i++)
1099		✗	ctx->next_prev[i] = ctx->next_prev[i + 1];
1100		✗	ctx->next_prev[i] = pic;
1101		✗	++pic->ref_count[0];
1102			}
1103			}
1104
1105		✗	static int vaapi_encode_pick_next(AVCodecContext *avctx,
1106			VAAPIEncodePicture **pic_out)
1107			{
1108		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
1109		✗	VAAPIEncodePicture *pic = NULL, *prev = NULL, *next, *start;
1110			int i, b_counter, closed_gop_end;
1111
1112			// If there are any B-frames already queued, the next one to encode
1113			// is the earliest not-yet-issued frame for which all references are
1114			// available.
1115		✗	for (pic = ctx->pic_start; pic; pic = pic->next) {
1116		✗	if (pic->encode_issued)
1117		✗	continue;
1118		✗	if (pic->type != PICTURE_TYPE_B)
1119		✗	continue;
1120		✗	for (i = 0; i < pic->nb_refs[0]; i++) {
1121		✗	if (!pic->refs[0][i]->encode_issued)
1122		✗	break;
1123			}
1124		✗	if (i != pic->nb_refs[0])
1125		✗	continue;
1126
1127		✗	for (i = 0; i < pic->nb_refs[1]; i++) {
1128		✗	if (!pic->refs[1][i]->encode_issued)
1129		✗	break;
1130			}
1131		✗	if (i == pic->nb_refs[1])
1132		✗	break;
1133			}
1134
1135		✗	if (pic) {
1136		✗	av_log(avctx, AV_LOG_DEBUG, "Pick B-picture at depth %d to "
1137			"encode next.\n", pic->b_depth);
1138		✗	*pic_out = pic;
1139		✗	return 0;
1140			}
1141
1142			// Find the B-per-Pth available picture to become the next picture
1143			// on the top layer.
1144		✗	start = NULL;
1145		✗	b_counter = 0;
1146		✗	closed_gop_end = ctx->closed_gop ||
1147		✗	ctx->idr_counter == ctx->gop_per_idr;
1148		✗	for (pic = ctx->pic_start; pic; pic = next) {
1149		✗	next = pic->next;
1150		✗	if (pic->encode_issued) {
1151		✗	start = pic;
1152		✗	continue;
1153			}
1154			// If the next available picture is force-IDR, encode it to start
1155			// a new GOP immediately.
1156		✗	if (pic->force_idr)
1157		✗	break;
1158		✗	if (b_counter == ctx->b_per_p)
1159		✗	break;
1160			// If this picture ends a closed GOP or starts a new GOP then it
1161			// needs to be in the top layer.
1162		✗	if (ctx->gop_counter + b_counter + closed_gop_end >= ctx->gop_size)
1163		✗	break;
1164			// If the picture after this one is force-IDR, we need to encode
1165			// this one in the top layer.
1166		✗	if (next && next->force_idr)
1167		✗	break;
1168		✗	++b_counter;
1169			}
1170
1171			// At the end of the stream the last picture must be in the top layer.
1172		✗	if (!pic && ctx->end_of_stream) {
1173		✗	--b_counter;
1174		✗	pic = ctx->pic_end;
1175		✗	if (pic->encode_complete)
1176		✗	return AVERROR_EOF;
1177		✗	else if (pic->encode_issued)
1178		✗	return AVERROR(EAGAIN);
1179			}
1180
1181		✗	if (!pic) {
1182		✗	av_log(avctx, AV_LOG_DEBUG, "Pick nothing to encode next - "
1183			"need more input for reference pictures.\n");
1184		✗	return AVERROR(EAGAIN);
1185			}
1186		✗	if (ctx->input_order <= ctx->decode_delay && !ctx->end_of_stream) {
1187		✗	av_log(avctx, AV_LOG_DEBUG, "Pick nothing to encode next - "
1188			"need more input for timestamps.\n");
1189		✗	return AVERROR(EAGAIN);
1190			}
1191
1192		✗	if (pic->force_idr) {
1193		✗	av_log(avctx, AV_LOG_DEBUG, "Pick forced IDR-picture to "
1194			"encode next.\n");
1195		✗	pic->type = PICTURE_TYPE_IDR;
1196		✗	ctx->idr_counter = 1;
1197		✗	ctx->gop_counter = 1;
1198
1199		✗	} else if (ctx->gop_counter + b_counter >= ctx->gop_size) {
1200		✗	if (ctx->idr_counter == ctx->gop_per_idr) {
1201		✗	av_log(avctx, AV_LOG_DEBUG, "Pick new-GOP IDR-picture to "
1202			"encode next.\n");
1203		✗	pic->type = PICTURE_TYPE_IDR;
1204		✗	ctx->idr_counter = 1;
1205			} else {
1206		✗	av_log(avctx, AV_LOG_DEBUG, "Pick new-GOP I-picture to "
1207			"encode next.\n");
1208		✗	pic->type = PICTURE_TYPE_I;
1209		✗	++ctx->idr_counter;
1210			}
1211		✗	ctx->gop_counter = 1;
1212
1213			} else {
1214		✗	if (ctx->gop_counter + b_counter + closed_gop_end == ctx->gop_size) {
1215		✗	av_log(avctx, AV_LOG_DEBUG, "Pick group-end P-picture to "
1216			"encode next.\n");
1217			} else {
1218		✗	av_log(avctx, AV_LOG_DEBUG, "Pick normal P-picture to "
1219			"encode next.\n");
1220			}
1221		✗	pic->type = PICTURE_TYPE_P;
1222		✗	av_assert0(start);
1223		✗	ctx->gop_counter += 1 + b_counter;
1224			}
1225		✗	pic->is_reference = 1;
1226		✗	*pic_out = pic;
1227
1228		✗	vaapi_encode_add_ref(avctx, pic, pic, 0, 1, 0);
1229		✗	if (pic->type != PICTURE_TYPE_IDR) {
1230			// TODO: apply both previous and forward multi reference for all vaapi encoders.
1231			// And L0/L1 reference frame number can be set dynamically through query
1232			// VAConfigAttribEncMaxRefFrames attribute.
1233		✗	if (avctx->codec_id == AV_CODEC_ID_AV1) {
1234		✗	for (i = 0; i < ctx->nb_next_prev; i++)
1235		✗	vaapi_encode_add_ref(avctx, pic, ctx->next_prev[i],
1236		✗	pic->type == PICTURE_TYPE_P,
1237			b_counter > 0, 0);
1238			} else
1239		✗	vaapi_encode_add_ref(avctx, pic, start,
1240		✗	pic->type == PICTURE_TYPE_P,
1241			b_counter > 0, 0);
1242
1243		✗	vaapi_encode_add_ref(avctx, pic, ctx->next_prev[ctx->nb_next_prev - 1], 0, 0, 1);
1244			}
1245
1246		✗	if (b_counter > 0) {
1247		✗	vaapi_encode_set_b_pictures(avctx, start, pic, pic, 1,
1248			&prev);
1249			} else {
1250		✗	prev = pic;
1251			}
1252		✗	vaapi_encode_add_next_prev(avctx, prev);
1253
1254		✗	return 0;
1255			}
1256
1257		✗	static int vaapi_encode_clear_old(AVCodecContext *avctx)
1258			{
1259		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
1260			VAAPIEncodePicture *pic, *prev, *next;
1261
1262		✗	av_assert0(ctx->pic_start);
1263
1264			// Remove direct references once each picture is complete.
1265		✗	for (pic = ctx->pic_start; pic; pic = pic->next) {
1266		✗	if (pic->encode_complete && pic->next)
1267		✗	vaapi_encode_remove_refs(avctx, pic, 0);
1268			}
1269
1270			// Remove indirect references once a picture has no direct references.
1271		✗	for (pic = ctx->pic_start; pic; pic = pic->next) {
1272		✗	if (pic->encode_complete && pic->ref_count[0] == 0)
1273		✗	vaapi_encode_remove_refs(avctx, pic, 1);
1274			}
1275
1276			// Clear out all complete pictures with no remaining references.
1277		✗	prev = NULL;
1278		✗	for (pic = ctx->pic_start; pic; pic = next) {
1279		✗	next = pic->next;
1280		✗	if (pic->encode_complete && pic->ref_count[1] == 0) {
1281		✗	av_assert0(pic->ref_removed[0] && pic->ref_removed[1]);
1282		✗	if (prev)
1283		✗	prev->next = next;
1284			else
1285		✗	ctx->pic_start = next;
1286		✗	vaapi_encode_free(avctx, pic);
1287			} else {
1288		✗	prev = pic;
1289			}
1290			}
1291
1292		✗	return 0;
1293			}
1294
1295		✗	static int vaapi_encode_check_frame(AVCodecContext *avctx,
1296			const AVFrame *frame)
1297			{
1298		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
1299
1300		✗	if ((frame->crop_top || frame->crop_bottom ||
1301		✗	frame->crop_left || frame->crop_right) && !ctx->crop_warned) {
1302		✗	av_log(avctx, AV_LOG_WARNING, "Cropping information on input "
1303			"frames ignored due to lack of API support.\n");
1304		✗	ctx->crop_warned = 1;
1305			}
1306
1307		✗	if (!ctx->roi_allowed) {
1308			AVFrameSideData *sd =
1309		✗	av_frame_get_side_data(frame, AV_FRAME_DATA_REGIONS_OF_INTEREST);
1310
1311		✗	if (sd && !ctx->roi_warned) {
1312		✗	av_log(avctx, AV_LOG_WARNING, "ROI side data on input "
1313			"frames ignored due to lack of driver support.\n");
1314		✗	ctx->roi_warned = 1;
1315			}
1316			}
1317
1318		✗	return 0;
1319			}
1320
1321		✗	static int vaapi_encode_send_frame(AVCodecContext *avctx, AVFrame *frame)
1322			{
1323		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
1324			VAAPIEncodePicture *pic;
1325			int err;
1326
1327		✗	if (frame) {
1328		✗	av_log(avctx, AV_LOG_DEBUG, "Input frame: %ux%u (%"PRId64").\n",
1329			frame->width, frame->height, frame->pts);
1330
1331		✗	err = vaapi_encode_check_frame(avctx, frame);
1332		✗	if (err < 0)
1333		✗	return err;
1334
1335		✗	pic = vaapi_encode_alloc(avctx);
1336		✗	if (!pic)
1337		✗	return AVERROR(ENOMEM);
1338
1339		✗	pic->input_image = av_frame_alloc();
1340		✗	if (!pic->input_image) {
1341		✗	err = AVERROR(ENOMEM);
1342		✗	goto fail;
1343			}
1344
1345		✗	if (ctx->input_order == 0 || frame->pict_type == AV_PICTURE_TYPE_I)
1346		✗	pic->force_idr = 1;
1347
1348		✗	pic->input_surface = (VASurfaceID)(uintptr_t)frame->data[3];
1349		✗	pic->pts = frame->pts;
1350		✗	pic->duration = frame->duration;
1351
1352		✗	if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) {
1353		✗	err = av_buffer_replace(&pic->opaque_ref, frame->opaque_ref);
1354		✗	if (err < 0)
1355		✗	goto fail;
1356
1357		✗	pic->opaque = frame->opaque;
1358			}
1359
1360		✗	av_frame_move_ref(pic->input_image, frame);
1361
1362		✗	if (ctx->input_order == 0)
1363		✗	ctx->first_pts = pic->pts;
1364		✗	if (ctx->input_order == ctx->decode_delay)
1365		✗	ctx->dts_pts_diff = pic->pts - ctx->first_pts;
1366		✗	if (ctx->output_delay > 0)
1367		✗	ctx->ts_ring[ctx->input_order %
1368		✗	(3 * ctx->output_delay + ctx->async_depth)] = pic->pts;
1369
1370		✗	pic->display_order = ctx->input_order;
1371		✗	++ctx->input_order;
1372
1373		✗	if (ctx->pic_start) {
1374		✗	ctx->pic_end->next = pic;
1375		✗	ctx->pic_end = pic;
1376			} else {
1377		✗	ctx->pic_start = pic;
1378		✗	ctx->pic_end = pic;
1379			}
1380
1381			} else {
1382		✗	ctx->end_of_stream = 1;
1383
1384			// Fix timestamps if we hit end-of-stream before the initial decode
1385			// delay has elapsed.
1386		✗	if (ctx->input_order < ctx->decode_delay)
1387		✗	ctx->dts_pts_diff = ctx->pic_end->pts - ctx->first_pts;
1388			}
1389
1390		✗	return 0;
1391
1392		✗	fail:
1393		✗	vaapi_encode_free(avctx, pic);
1394		✗	return err;
1395			}
1396
1397		✗	int ff_vaapi_encode_receive_packet(AVCodecContext *avctx, AVPacket *pkt)
1398			{
1399		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
1400		✗	VAAPIEncodePicture *pic = NULL;
1401		✗	AVFrame *frame = ctx->frame;
1402			int err;
1403
1404		✗	start:
1405			/** if no B frame before repeat P frame, sent repeat P frame out. */
1406		✗	if (ctx->tail_pkt->size) {
1407		✗	for (VAAPIEncodePicture *tmp = ctx->pic_start; tmp; tmp = tmp->next) {
1408		✗	if (tmp->type == PICTURE_TYPE_B && tmp->pts < ctx->tail_pkt->pts)
1409			break;
1410		✗	else if (!tmp->next) {
1411		✗	av_packet_move_ref(pkt, ctx->tail_pkt);
1412		✗	goto end;
1413			}
1414			}
1415			}
1416
1417		✗	err = ff_encode_get_frame(avctx, frame);
1418		✗	if (err < 0 && err != AVERROR_EOF)
1419		✗	return err;
1420
1421		✗	if (err == AVERROR_EOF)
1422		✗	frame = NULL;
1423
1424		✗	err = vaapi_encode_send_frame(avctx, frame);
1425		✗	if (err < 0)
1426		✗	return err;
1427
1428		✗	if (!ctx->pic_start) {
1429		✗	if (ctx->end_of_stream)
1430		✗	return AVERROR_EOF;
1431			else
1432		✗	return AVERROR(EAGAIN);
1433			}
1434
1435		✗	if (ctx->has_sync_buffer_func) {
1436		✗	if (av_fifo_can_write(ctx->encode_fifo)) {
1437		✗	err = vaapi_encode_pick_next(avctx, &pic);
1438		✗	if (!err) {
1439		✗	av_assert0(pic);
1440		✗	pic->encode_order = ctx->encode_order +
1441		✗	av_fifo_can_read(ctx->encode_fifo);
1442		✗	err = vaapi_encode_issue(avctx, pic);
1443		✗	if (err < 0) {
1444		✗	av_log(avctx, AV_LOG_ERROR, "Encode failed: %d.\n", err);
1445		✗	return err;
1446			}
1447		✗	av_fifo_write(ctx->encode_fifo, &pic, 1);
1448			}
1449			}
1450
1451		✗	if (!av_fifo_can_read(ctx->encode_fifo))
1452		✗	return err;
1453
1454			// More frames can be buffered
1455		✗	if (av_fifo_can_write(ctx->encode_fifo) && !ctx->end_of_stream)
1456		✗	return AVERROR(EAGAIN);
1457
1458		✗	av_fifo_read(ctx->encode_fifo, &pic, 1);
1459		✗	ctx->encode_order = pic->encode_order + 1;
1460			} else {
1461		✗	err = vaapi_encode_pick_next(avctx, &pic);
1462		✗	if (err < 0)
1463		✗	return err;
1464		✗	av_assert0(pic);
1465
1466		✗	pic->encode_order = ctx->encode_order++;
1467
1468		✗	err = vaapi_encode_issue(avctx, pic);
1469		✗	if (err < 0) {
1470		✗	av_log(avctx, AV_LOG_ERROR, "Encode failed: %d.\n", err);
1471		✗	return err;
1472			}
1473			}
1474
1475		✗	err = vaapi_encode_output(avctx, pic, pkt);
1476		✗	if (err < 0) {
1477		✗	av_log(avctx, AV_LOG_ERROR, "Output failed: %d.\n", err);
1478		✗	return err;
1479			}
1480
1481		✗	ctx->output_order = pic->encode_order;
1482		✗	vaapi_encode_clear_old(avctx);
1483
1484			/** loop to get an available pkt in encoder flushing. */
1485		✗	if (ctx->end_of_stream && !pkt->size)
1486		✗	goto start;
1487
1488		✗	end:
1489		✗	if (pkt->size)
1490		✗	av_log(avctx, AV_LOG_DEBUG, "Output packet: pts %"PRId64", dts %"PRId64", "
1491			"size %d bytes.\n", pkt->pts, pkt->dts, pkt->size);
1492
1493		✗	return 0;
1494			}
1495
1496		✗	static av_cold void vaapi_encode_add_global_param(AVCodecContext *avctx, int type,
1497			void *buffer, size_t size)
1498			{
1499		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
1500
1501		✗	av_assert0(ctx->nb_global_params < MAX_GLOBAL_PARAMS);
1502
1503		✗	ctx->global_params_type[ctx->nb_global_params] = type;
1504		✗	ctx->global_params [ctx->nb_global_params] = buffer;
1505		✗	ctx->global_params_size[ctx->nb_global_params] = size;
1506
1507		✗	++ctx->nb_global_params;
1508		✗	}
1509
1510			typedef struct VAAPIEncodeRTFormat {
1511			const char *name;
1512			unsigned int value;
1513			int depth;
1514			int nb_components;
1515			int log2_chroma_w;
1516			int log2_chroma_h;
1517			} VAAPIEncodeRTFormat;
1518
1519			static const VAAPIEncodeRTFormat vaapi_encode_rt_formats[] = {
1520			{ "YUV400", VA_RT_FORMAT_YUV400, 8, 1, },
1521			{ "YUV420", VA_RT_FORMAT_YUV420, 8, 3, 1, 1 },
1522			{ "YUV422", VA_RT_FORMAT_YUV422, 8, 3, 1, 0 },
1523			#if VA_CHECK_VERSION(1, 2, 0)
1524			{ "YUV420_12", VA_RT_FORMAT_YUV420_12, 12, 3, 1, 1 },
1525			{ "YUV422_10", VA_RT_FORMAT_YUV422_10, 10, 3, 1, 0 },
1526			{ "YUV422_12", VA_RT_FORMAT_YUV422_12, 12, 3, 1, 0 },
1527			{ "YUV444_10", VA_RT_FORMAT_YUV444_10, 10, 3, 0, 0 },
1528			{ "YUV444_12", VA_RT_FORMAT_YUV444_12, 12, 3, 0, 0 },
1529			#endif
1530			{ "YUV444", VA_RT_FORMAT_YUV444, 8, 3, 0, 0 },
1531			{ "XYUV", VA_RT_FORMAT_YUV444, 8, 3, 0, 0 },
1532			{ "YUV411", VA_RT_FORMAT_YUV411, 8, 3, 2, 0 },
1533			#if VA_CHECK_VERSION(0, 38, 1)
1534			{ "YUV420_10", VA_RT_FORMAT_YUV420_10BPP, 10, 3, 1, 1 },
1535			#endif
1536			};
1537
1538			static const VAEntrypoint vaapi_encode_entrypoints_normal[] = {
1539			VAEntrypointEncSlice,
1540			VAEntrypointEncPicture,
1541			#if VA_CHECK_VERSION(0, 39, 2)
1542			VAEntrypointEncSliceLP,
1543			#endif
1544			0
1545			};
1546			#if VA_CHECK_VERSION(0, 39, 2)
1547			static const VAEntrypoint vaapi_encode_entrypoints_low_power[] = {
1548			VAEntrypointEncSliceLP,
1549			0
1550			};
1551			#endif
1552
1553		✗	static av_cold int vaapi_encode_profile_entrypoint(AVCodecContext *avctx)
1554			{
1555		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
1556		✗	VAProfile *va_profiles = NULL;
1557		✗	VAEntrypoint *va_entrypoints = NULL;
1558			VAStatus vas;
1559			const VAEntrypoint *usable_entrypoints;
1560			const VAAPIEncodeProfile *profile;
1561			const AVPixFmtDescriptor *desc;
1562			VAConfigAttrib rt_format_attr;
1563			const VAAPIEncodeRTFormat *rt_format;
1564			const char *profile_string, *entrypoint_string;
1565			int i, j, n, depth, err;
1566
1567
1568		✗	if (ctx->low_power) {
1569			#if VA_CHECK_VERSION(0, 39, 2)
1570		✗	usable_entrypoints = vaapi_encode_entrypoints_low_power;
1571			#else
1572			av_log(avctx, AV_LOG_ERROR, "Low-power encoding is not "
1573			"supported with this VAAPI version.\n");
1574			return AVERROR(EINVAL);
1575			#endif
1576			} else {
1577		✗	usable_entrypoints = vaapi_encode_entrypoints_normal;
1578			}
1579
1580		✗	desc = av_pix_fmt_desc_get(ctx->input_frames->sw_format);
1581		✗	if (!desc) {
1582		✗	av_log(avctx, AV_LOG_ERROR, "Invalid input pixfmt (%d).\n",
1583		✗	ctx->input_frames->sw_format);
1584		✗	return AVERROR(EINVAL);
1585			}
1586		✗	depth = desc->comp[0].depth;
1587		✗	for (i = 1; i < desc->nb_components; i++) {
1588		✗	if (desc->comp[i].depth != depth) {
1589		✗	av_log(avctx, AV_LOG_ERROR, "Invalid input pixfmt (%s).\n",
1590		✗	desc->name);
1591		✗	return AVERROR(EINVAL);
1592			}
1593			}
1594		✗	av_log(avctx, AV_LOG_VERBOSE, "Input surface format is %s.\n",
1595		✗	desc->name);
1596
1597		✗	n = vaMaxNumProfiles(ctx->hwctx->display);
1598		✗	va_profiles = av_malloc_array(n, sizeof(VAProfile));
1599		✗	if (!va_profiles) {
1600		✗	err = AVERROR(ENOMEM);
1601		✗	goto fail;
1602			}
1603		✗	vas = vaQueryConfigProfiles(ctx->hwctx->display, va_profiles, &n);
1604		✗	if (vas != VA_STATUS_SUCCESS) {
1605		✗	av_log(avctx, AV_LOG_ERROR, "Failed to query profiles: %d (%s).\n",
1606			vas, vaErrorStr(vas));
1607		✗	err = AVERROR_EXTERNAL;
1608		✗	goto fail;
1609			}
1610
1611		✗	av_assert0(ctx->codec->profiles);
1612		✗	for (i = 0; (ctx->codec->profiles[i].av_profile !=
1613		✗	AV_PROFILE_UNKNOWN); i++) {
1614		✗	profile = &ctx->codec->profiles[i];
1615		✗	if (depth != profile->depth ||
1616		✗	desc->nb_components != profile->nb_components)
1617		✗	continue;
1618		✗	if (desc->nb_components > 1 &&
1619		✗	(desc->log2_chroma_w != profile->log2_chroma_w ||
1620		✗	desc->log2_chroma_h != profile->log2_chroma_h))
1621		✗	continue;
1622		✗	if (avctx->profile != profile->av_profile &&
1623		✗	avctx->profile != AV_PROFILE_UNKNOWN)
1624		✗	continue;
1625
1626			#if VA_CHECK_VERSION(1, 0, 0)
1627		✗	profile_string = vaProfileStr(profile->va_profile);
1628			#else
1629			profile_string = "(no profile names)";
1630			#endif
1631
1632		✗	for (j = 0; j < n; j++) {
1633		✗	if (va_profiles[j] == profile->va_profile)
1634		✗	break;
1635			}
1636		✗	if (j >= n) {
1637		✗	av_log(avctx, AV_LOG_VERBOSE, "Compatible profile %s (%d) "
1638			"is not supported by driver.\n", profile_string,
1639		✗	profile->va_profile);
1640		✗	continue;
1641			}
1642
1643		✗	ctx->profile = profile;
1644		✗	break;
1645			}
1646		✗	if (!ctx->profile) {
1647		✗	av_log(avctx, AV_LOG_ERROR, "No usable encoding profile found.\n");
1648		✗	err = AVERROR(ENOSYS);
1649		✗	goto fail;
1650			}
1651
1652		✗	avctx->profile = profile->av_profile;
1653		✗	ctx->va_profile = profile->va_profile;
1654		✗	av_log(avctx, AV_LOG_VERBOSE, "Using VAAPI profile %s (%d).\n",
1655		✗	profile_string, ctx->va_profile);
1656
1657		✗	n = vaMaxNumEntrypoints(ctx->hwctx->display);
1658		✗	va_entrypoints = av_malloc_array(n, sizeof(VAEntrypoint));
1659		✗	if (!va_entrypoints) {
1660		✗	err = AVERROR(ENOMEM);
1661		✗	goto fail;
1662			}
1663		✗	vas = vaQueryConfigEntrypoints(ctx->hwctx->display, ctx->va_profile,
1664			va_entrypoints, &n);
1665		✗	if (vas != VA_STATUS_SUCCESS) {
1666		✗	av_log(avctx, AV_LOG_ERROR, "Failed to query entrypoints for "
1667			"profile %s (%d): %d (%s).\n", profile_string,
1668		✗	ctx->va_profile, vas, vaErrorStr(vas));
1669		✗	err = AVERROR_EXTERNAL;
1670		✗	goto fail;
1671			}
1672
1673		✗	for (i = 0; i < n; i++) {
1674		✗	for (j = 0; usable_entrypoints[j]; j++) {
1675		✗	if (va_entrypoints[i] == usable_entrypoints[j])
1676		✗	break;
1677			}
1678		✗	if (usable_entrypoints[j])
1679		✗	break;
1680			}
1681		✗	if (i >= n) {
1682		✗	av_log(avctx, AV_LOG_ERROR, "No usable encoding entrypoint found "
1683		✗	"for profile %s (%d).\n", profile_string, ctx->va_profile);
1684		✗	err = AVERROR(ENOSYS);
1685		✗	goto fail;
1686			}
1687
1688		✗	ctx->va_entrypoint = va_entrypoints[i];
1689			#if VA_CHECK_VERSION(1, 0, 0)
1690		✗	entrypoint_string = vaEntrypointStr(ctx->va_entrypoint);
1691			#else
1692			entrypoint_string = "(no entrypoint names)";
1693			#endif
1694		✗	av_log(avctx, AV_LOG_VERBOSE, "Using VAAPI entrypoint %s (%d).\n",
1695		✗	entrypoint_string, ctx->va_entrypoint);
1696
1697		✗	for (i = 0; i < FF_ARRAY_ELEMS(vaapi_encode_rt_formats); i++) {
1698		✗	rt_format = &vaapi_encode_rt_formats[i];
1699		✗	if (rt_format->depth == depth &&
1700		✗	rt_format->nb_components == profile->nb_components &&
1701		✗	rt_format->log2_chroma_w == profile->log2_chroma_w &&
1702		✗	rt_format->log2_chroma_h == profile->log2_chroma_h)
1703		✗	break;
1704			}
1705		✗	if (i >= FF_ARRAY_ELEMS(vaapi_encode_rt_formats)) {
1706		✗	av_log(avctx, AV_LOG_ERROR, "No usable render target format "
1707			"found for profile %s (%d) entrypoint %s (%d).\n",
1708		✗	profile_string, ctx->va_profile,
1709		✗	entrypoint_string, ctx->va_entrypoint);
1710		✗	err = AVERROR(ENOSYS);
1711		✗	goto fail;
1712			}
1713
1714		✗	rt_format_attr = (VAConfigAttrib) { VAConfigAttribRTFormat };
1715		✗	vas = vaGetConfigAttributes(ctx->hwctx->display,
1716			ctx->va_profile, ctx->va_entrypoint,
1717			&rt_format_attr, 1);
1718		✗	if (vas != VA_STATUS_SUCCESS) {
1719		✗	av_log(avctx, AV_LOG_ERROR, "Failed to query RT format "
1720			"config attribute: %d (%s).\n", vas, vaErrorStr(vas));
1721		✗	err = AVERROR_EXTERNAL;
1722		✗	goto fail;
1723			}
1724
1725		✗	if (rt_format_attr.value == VA_ATTRIB_NOT_SUPPORTED) {
1726		✗	av_log(avctx, AV_LOG_VERBOSE, "RT format config attribute not "
1727			"supported by driver: assuming surface RT format %s "
1728		✗	"is valid.\n", rt_format->name);
1729		✗	} else if (!(rt_format_attr.value & rt_format->value)) {
1730		✗	av_log(avctx, AV_LOG_ERROR, "Surface RT format %s not supported "
1731			"by driver for encoding profile %s (%d) entrypoint %s (%d).\n",
1732		✗	rt_format->name, profile_string, ctx->va_profile,
1733		✗	entrypoint_string, ctx->va_entrypoint);
1734		✗	err = AVERROR(ENOSYS);
1735		✗	goto fail;
1736			} else {
1737		✗	av_log(avctx, AV_LOG_VERBOSE, "Using VAAPI render target "
1738		✗	"format %s (%#x).\n", rt_format->name, rt_format->value);
1739		✗	ctx->config_attributes[ctx->nb_config_attributes++] =
1740			(VAConfigAttrib) {
1741			.type = VAConfigAttribRTFormat,
1742		✗	.value = rt_format->value,
1743			};
1744			}
1745
1746		✗	err = 0;
1747		✗	fail:
1748		✗	av_freep(&va_profiles);
1749		✗	av_freep(&va_entrypoints);
1750		✗	return err;
1751			}
1752
1753			static const VAAPIEncodeRCMode vaapi_encode_rc_modes[] = {
1754			// Bitrate Quality
1755			// | Maxrate | HRD/VBV
1756			{ 0 }, // | | | |
1757			{ RC_MODE_CQP, "CQP", 1, VA_RC_CQP, 0, 0, 1, 0 },
1758			{ RC_MODE_CBR, "CBR", 1, VA_RC_CBR, 1, 0, 0, 1 },
1759			{ RC_MODE_VBR, "VBR", 1, VA_RC_VBR, 1, 1, 0, 1 },
1760			#if VA_CHECK_VERSION(1, 1, 0)
1761			{ RC_MODE_ICQ, "ICQ", 1, VA_RC_ICQ, 0, 0, 1, 0 },
1762			#else
1763			{ RC_MODE_ICQ, "ICQ", 0 },
1764			#endif
1765			#if VA_CHECK_VERSION(1, 3, 0)
1766			{ RC_MODE_QVBR, "QVBR", 1, VA_RC_QVBR, 1, 1, 1, 1 },
1767			{ RC_MODE_AVBR, "AVBR", 0, VA_RC_AVBR, 1, 0, 0, 0 },
1768			#else
1769			{ RC_MODE_QVBR, "QVBR", 0 },
1770			{ RC_MODE_AVBR, "AVBR", 0 },
1771			#endif
1772			};
1773
1774		✗	static av_cold int vaapi_encode_init_rate_control(AVCodecContext *avctx)
1775			{
1776		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
1777			uint32_t supported_va_rc_modes;
1778			const VAAPIEncodeRCMode *rc_mode;
1779			int64_t rc_bits_per_second;
1780			int rc_target_percentage;
1781			int rc_window_size;
1782			int rc_quality;
1783			int64_t hrd_buffer_size;
1784			int64_t hrd_initial_buffer_fullness;
1785			int fr_num, fr_den;
1786		✗	VAConfigAttrib rc_attr = { VAConfigAttribRateControl };
1787			VAStatus vas;
1788			char supported_rc_modes_string[64];
1789
1790		✗	vas = vaGetConfigAttributes(ctx->hwctx->display,
1791			ctx->va_profile, ctx->va_entrypoint,
1792			&rc_attr, 1);
1793		✗	if (vas != VA_STATUS_SUCCESS) {
1794		✗	av_log(avctx, AV_LOG_ERROR, "Failed to query rate control "
1795			"config attribute: %d (%s).\n", vas, vaErrorStr(vas));
1796		✗	return AVERROR_EXTERNAL;
1797			}
1798		✗	if (rc_attr.value == VA_ATTRIB_NOT_SUPPORTED) {
1799		✗	av_log(avctx, AV_LOG_VERBOSE, "Driver does not report any "
1800			"supported rate control modes: assuming CQP only.\n");
1801		✗	supported_va_rc_modes = VA_RC_CQP;
1802		✗	strcpy(supported_rc_modes_string, "unknown");
1803			} else {
1804		✗	char *str = supported_rc_modes_string;
1805		✗	size_t len = sizeof(supported_rc_modes_string);
1806		✗	int i, first = 1, res;
1807
1808		✗	supported_va_rc_modes = rc_attr.value;
1809		✗	if (ctx->blbrc) {
1810			#if VA_CHECK_VERSION(0, 39, 2)
1811		✗	if (!(supported_va_rc_modes & VA_RC_MB)) {
1812		✗	ctx->blbrc = 0;
1813		✗	av_log(avctx, AV_LOG_WARNING, "Driver does not support BLBRC.\n");
1814			}
1815			#else
1816			ctx->blbrc = 0;
1817			av_log(avctx, AV_LOG_WARNING, "Please consider to update to VAAPI 0.39.2 "
1818			"or above, which can support BLBRC.\n");
1819			#endif
1820			}
1821
1822		✗	for (i = 0; i < FF_ARRAY_ELEMS(vaapi_encode_rc_modes); i++) {
1823		✗	rc_mode = &vaapi_encode_rc_modes[i];
1824		✗	if (supported_va_rc_modes & rc_mode->va_mode) {
1825		✗	res = snprintf(str, len, "%s%s",
1826		✗	first ? "" : ", ", rc_mode->name);
1827		✗	first = 0;
1828		✗	if (res < 0) {
1829		✗	*str = 0;
1830		✗	break;
1831			}
1832		✗	len -= res;
1833		✗	str += res;
1834		✗	if (len == 0)
1835		✗	break;
1836			}
1837			}
1838
1839		✗	av_log(avctx, AV_LOG_DEBUG, "Driver supports RC modes %s.\n",
1840			supported_rc_modes_string);
1841			}
1842
1843			// Rate control mode selection:
1844			// * If the user has set a mode explicitly with the rc_mode option,
1845			// use it and fail if it is not available.
1846			// * If an explicit QP option has been set, use CQP.
1847			// * If the codec is CQ-only, use CQP.
1848			// * If the QSCALE avcodec option is set, use CQP.
1849			// * If bitrate and quality are both set, try QVBR.
1850			// * If quality is set, try ICQ, then CQP.
1851			// * If bitrate and maxrate are set and have the same value, try CBR.
1852			// * If a bitrate is set, try AVBR, then VBR, then CBR.
1853			// * If no bitrate is set, try ICQ, then CQP.
1854
1855			#define TRY_RC_MODE(mode, fail) do { \
1856			rc_mode = &vaapi_encode_rc_modes[mode]; \
1857			if (!(rc_mode->va_mode & supported_va_rc_modes)) { \
1858			if (fail) { \
1859			av_log(avctx, AV_LOG_ERROR, "Driver does not support %s " \
1860			"RC mode (supported modes: %s).\n", rc_mode->name, \
1861			supported_rc_modes_string); \
1862			return AVERROR(EINVAL); \
1863			} \
1864			av_log(avctx, AV_LOG_DEBUG, "Driver does not support %s " \
1865			"RC mode.\n", rc_mode->name); \
1866			rc_mode = NULL; \
1867			} else { \
1868			goto rc_mode_found; \
1869			} \
1870			} while (0)
1871
1872		✗	if (ctx->explicit_rc_mode)
1873		✗	TRY_RC_MODE(ctx->explicit_rc_mode, 1);
1874
1875		✗	if (ctx->explicit_qp)
1876		✗	TRY_RC_MODE(RC_MODE_CQP, 1);
1877
1878		✗	if (ctx->codec->flags & FLAG_CONSTANT_QUALITY_ONLY)
1879		✗	TRY_RC_MODE(RC_MODE_CQP, 1);
1880
1881		✗	if (avctx->flags & AV_CODEC_FLAG_QSCALE)
1882		✗	TRY_RC_MODE(RC_MODE_CQP, 1);
1883
1884		✗	if (avctx->bit_rate > 0 && avctx->global_quality > 0)
1885		✗	TRY_RC_MODE(RC_MODE_QVBR, 0);
1886
1887		✗	if (avctx->global_quality > 0) {
1888		✗	TRY_RC_MODE(RC_MODE_ICQ, 0);
1889		✗	TRY_RC_MODE(RC_MODE_CQP, 0);
1890			}
1891
1892		✗	if (avctx->bit_rate > 0 && avctx->rc_max_rate == avctx->bit_rate)
1893		✗	TRY_RC_MODE(RC_MODE_CBR, 0);
1894
1895		✗	if (avctx->bit_rate > 0) {
1896		✗	TRY_RC_MODE(RC_MODE_AVBR, 0);
1897		✗	TRY_RC_MODE(RC_MODE_VBR, 0);
1898		✗	TRY_RC_MODE(RC_MODE_CBR, 0);
1899			} else {
1900		✗	TRY_RC_MODE(RC_MODE_ICQ, 0);
1901		✗	TRY_RC_MODE(RC_MODE_CQP, 0);
1902			}
1903
1904		✗	av_log(avctx, AV_LOG_ERROR, "Driver does not support any "
1905			"RC mode compatible with selected options "
1906			"(supported modes: %s).\n", supported_rc_modes_string);
1907		✗	return AVERROR(EINVAL);
1908
1909		✗	rc_mode_found:
1910		✗	if (rc_mode->bitrate) {
1911		✗	if (avctx->bit_rate <= 0) {
1912		✗	av_log(avctx, AV_LOG_ERROR, "Bitrate must be set for %s "
1913		✗	"RC mode.\n", rc_mode->name);
1914		✗	return AVERROR(EINVAL);
1915			}
1916
1917		✗	if (rc_mode->mode == RC_MODE_AVBR) {
1918			// For maximum confusion AVBR is hacked into the existing API
1919			// by overloading some of the fields with completely different
1920			// meanings.
1921
1922			// Target percentage does not apply in AVBR mode.
1923		✗	rc_bits_per_second = avctx->bit_rate;
1924
1925			// Accuracy tolerance range for meeting the specified target
1926			// bitrate. It's very unclear how this is actually intended
1927			// to work - since we do want to get the specified bitrate,
1928			// set the accuracy to 100% for now.
1929		✗	rc_target_percentage = 100;
1930
1931			// Convergence period in frames. The GOP size reflects the
1932			// user's intended block size for cutting, so reusing that
1933			// as the convergence period seems a reasonable default.
1934		✗	rc_window_size = avctx->gop_size > 0 ? avctx->gop_size : 60;
1935
1936		✗	} else if (rc_mode->maxrate) {
1937		✗	if (avctx->rc_max_rate > 0) {
1938		✗	if (avctx->rc_max_rate < avctx->bit_rate) {
1939		✗	av_log(avctx, AV_LOG_ERROR, "Invalid bitrate settings: "
1940			"bitrate (%"PRId64") must not be greater than "
1941			"maxrate (%"PRId64").\n", avctx->bit_rate,
1942			avctx->rc_max_rate);
1943		✗	return AVERROR(EINVAL);
1944			}
1945		✗	rc_bits_per_second = avctx->rc_max_rate;
1946		✗	rc_target_percentage = (avctx->bit_rate * 100) /
1947		✗	avctx->rc_max_rate;
1948			} else {
1949			// We only have a target bitrate, but this mode requires
1950			// that a maximum rate be supplied as well. Since the
1951			// user does not want this to be a constraint, arbitrarily
1952			// pick a maximum rate of double the target rate.
1953		✗	rc_bits_per_second = 2 * avctx->bit_rate;
1954		✗	rc_target_percentage = 50;
1955			}
1956			} else {
1957		✗	if (avctx->rc_max_rate > avctx->bit_rate) {
1958		✗	av_log(avctx, AV_LOG_WARNING, "Max bitrate is ignored "
1959		✗	"in %s RC mode.\n", rc_mode->name);
1960			}
1961		✗	rc_bits_per_second = avctx->bit_rate;
1962		✗	rc_target_percentage = 100;
1963			}
1964			} else {
1965		✗	rc_bits_per_second = 0;
1966		✗	rc_target_percentage = 100;
1967			}
1968
1969		✗	if (rc_mode->quality) {
1970		✗	if (ctx->explicit_qp) {
1971		✗	rc_quality = ctx->explicit_qp;
1972		✗	} else if (avctx->global_quality > 0) {
1973		✗	if (avctx->flags & AV_CODEC_FLAG_QSCALE)
1974		✗	rc_quality = avctx->global_quality / FF_QP2LAMBDA;
1975			else
1976		✗	rc_quality = avctx->global_quality;
1977			} else {
1978		✗	rc_quality = ctx->codec->default_quality;
1979		✗	av_log(avctx, AV_LOG_WARNING, "No quality level set; "
1980			"using default (%d).\n", rc_quality);
1981			}
1982			} else {
1983		✗	rc_quality = 0;
1984			}
1985
1986		✗	if (rc_mode->hrd) {
1987		✗	if (avctx->rc_buffer_size)
1988		✗	hrd_buffer_size = avctx->rc_buffer_size;
1989		✗	else if (avctx->rc_max_rate > 0)
1990		✗	hrd_buffer_size = avctx->rc_max_rate;
1991			else
1992		✗	hrd_buffer_size = avctx->bit_rate;
1993		✗	if (avctx->rc_initial_buffer_occupancy) {
1994		✗	if (avctx->rc_initial_buffer_occupancy > hrd_buffer_size) {
1995		✗	av_log(avctx, AV_LOG_ERROR, "Invalid RC buffer settings: "
1996			"must have initial buffer size (%d) <= "
1997			"buffer size (%"PRId64").\n",
1998			avctx->rc_initial_buffer_occupancy, hrd_buffer_size);
1999		✗	return AVERROR(EINVAL);
2000			}
2001		✗	hrd_initial_buffer_fullness = avctx->rc_initial_buffer_occupancy;
2002			} else {
2003		✗	hrd_initial_buffer_fullness = hrd_buffer_size * 3 / 4;
2004			}
2005
2006		✗	rc_window_size = (hrd_buffer_size * 1000) / rc_bits_per_second;
2007			} else {
2008		✗	if (avctx->rc_buffer_size || avctx->rc_initial_buffer_occupancy) {
2009		✗	av_log(avctx, AV_LOG_WARNING, "Buffering settings are ignored "
2010		✗	"in %s RC mode.\n", rc_mode->name);
2011			}
2012
2013		✗	hrd_buffer_size = 0;
2014		✗	hrd_initial_buffer_fullness = 0;
2015
2016		✗	if (rc_mode->mode != RC_MODE_AVBR) {
2017			// Already set (with completely different meaning) for AVBR.
2018		✗	rc_window_size = 1000;
2019			}
2020			}
2021
2022		✗	if (rc_bits_per_second > UINT32_MAX ||
2023		✗	hrd_buffer_size > UINT32_MAX ||
2024			hrd_initial_buffer_fullness > UINT32_MAX) {
2025		✗	av_log(avctx, AV_LOG_ERROR, "RC parameters of 2^32 or "
2026			"greater are not supported by VAAPI.\n");
2027		✗	return AVERROR(EINVAL);
2028			}
2029
2030		✗	ctx->rc_mode = rc_mode;
2031		✗	ctx->rc_quality = rc_quality;
2032		✗	ctx->va_rc_mode = rc_mode->va_mode;
2033		✗	ctx->va_bit_rate = rc_bits_per_second;
2034
2035		✗	av_log(avctx, AV_LOG_VERBOSE, "RC mode: %s.\n", rc_mode->name);
2036
2037		✗	if (ctx->blbrc && ctx->va_rc_mode == VA_RC_CQP)
2038		✗	ctx->blbrc = 0;
2039		✗	av_log(avctx, AV_LOG_VERBOSE, "Block Level bitrate control: %s.\n", ctx->blbrc ? "ON" : "OFF");
2040
2041		✗	if (rc_attr.value == VA_ATTRIB_NOT_SUPPORTED) {
2042			// This driver does not want the RC mode attribute to be set.
2043			} else {
2044		✗	ctx->config_attributes[ctx->nb_config_attributes++] =
2045			(VAConfigAttrib) {
2046			.type = VAConfigAttribRateControl,
2047			#if VA_CHECK_VERSION(0, 39, 2)
2048		✗	.value = ctx->blbrc ? ctx->va_rc_mode | VA_RC_MB : ctx->va_rc_mode,
2049			#else
2050			.value = ctx->va_rc_mode,
2051			#endif
2052			};
2053			}
2054
2055		✗	if (rc_mode->quality)
2056		✗	av_log(avctx, AV_LOG_VERBOSE, "RC quality: %d.\n", rc_quality);
2057
2058		✗	if (rc_mode->va_mode != VA_RC_CQP) {
2059		✗	if (rc_mode->mode == RC_MODE_AVBR) {
2060		✗	av_log(avctx, AV_LOG_VERBOSE, "RC target: %"PRId64" bps "
2061			"converging in %d frames with %d%% accuracy.\n",
2062			rc_bits_per_second, rc_window_size,
2063			rc_target_percentage);
2064		✗	} else if (rc_mode->bitrate) {
2065		✗	av_log(avctx, AV_LOG_VERBOSE, "RC target: %d%% of "
2066			"%"PRId64" bps over %d ms.\n", rc_target_percentage,
2067			rc_bits_per_second, rc_window_size);
2068			}
2069
2070		✗	ctx->rc_params = (VAEncMiscParameterRateControl) {
2071			.bits_per_second = rc_bits_per_second,
2072			.target_percentage = rc_target_percentage,
2073			.window_size = rc_window_size,
2074			.initial_qp = 0,
2075		✗	.min_qp = (avctx->qmin > 0 ? avctx->qmin : 0),
2076			.basic_unit_size = 0,
2077			#if VA_CHECK_VERSION(1, 1, 0)
2078		✗	.ICQ_quality_factor = av_clip(rc_quality, 1, 51),
2079		✗	.max_qp = (avctx->qmax > 0 ? avctx->qmax : 0),
2080			#endif
2081			#if VA_CHECK_VERSION(1, 3, 0)
2082			.quality_factor = rc_quality,
2083			#endif
2084			#if VA_CHECK_VERSION(0, 39, 2)
2085		✗	.rc_flags.bits.mb_rate_control = ctx->blbrc ? 1 : 2,
2086			#endif
2087			};
2088		✗	vaapi_encode_add_global_param(avctx,
2089			VAEncMiscParameterTypeRateControl,
2090		✗	&ctx->rc_params,
2091			sizeof(ctx->rc_params));
2092			}
2093
2094		✗	if (rc_mode->hrd) {
2095		✗	av_log(avctx, AV_LOG_VERBOSE, "RC buffer: %"PRId64" bits, "
2096			"initial fullness %"PRId64" bits.\n",
2097			hrd_buffer_size, hrd_initial_buffer_fullness);
2098
2099		✗	ctx->hrd_params = (VAEncMiscParameterHRD) {
2100			.initial_buffer_fullness = hrd_initial_buffer_fullness,
2101			.buffer_size = hrd_buffer_size,
2102			};
2103		✗	vaapi_encode_add_global_param(avctx,
2104			VAEncMiscParameterTypeHRD,
2105		✗	&ctx->hrd_params,
2106			sizeof(ctx->hrd_params));
2107			}
2108
2109		✗	if (avctx->framerate.num > 0 && avctx->framerate.den > 0)
2110		✗	av_reduce(&fr_num, &fr_den,
2111		✗	avctx->framerate.num, avctx->framerate.den, 65535);
2112			else
2113		✗	av_reduce(&fr_num, &fr_den,
2114		✗	avctx->time_base.den, avctx->time_base.num, 65535);
2115
2116		✗	av_log(avctx, AV_LOG_VERBOSE, "RC framerate: %d/%d (%.2f fps).\n",
2117		✗	fr_num, fr_den, (double)fr_num / fr_den);
2118
2119		✗	ctx->fr_params = (VAEncMiscParameterFrameRate) {
2120		✗	.framerate = (unsigned int)fr_den << 16 | fr_num,
2121			};
2122			#if VA_CHECK_VERSION(0, 40, 0)
2123		✗	vaapi_encode_add_global_param(avctx,
2124			VAEncMiscParameterTypeFrameRate,
2125		✗	&ctx->fr_params,
2126			sizeof(ctx->fr_params));
2127			#endif
2128
2129		✗	return 0;
2130			}
2131
2132		✗	static av_cold int vaapi_encode_init_max_frame_size(AVCodecContext *avctx)
2133			{
2134			#if VA_CHECK_VERSION(1, 5, 0)
2135		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2136		✗	VAConfigAttrib attr = { VAConfigAttribMaxFrameSize };
2137			VAStatus vas;
2138
2139		✗	if (ctx->va_rc_mode == VA_RC_CQP) {
2140		✗	ctx->max_frame_size = 0;
2141		✗	av_log(avctx, AV_LOG_ERROR, "Max frame size is invalid in CQP rate "
2142			"control mode.\n");
2143		✗	return AVERROR(EINVAL);
2144			}
2145
2146		✗	vas = vaGetConfigAttributes(ctx->hwctx->display,
2147			ctx->va_profile,
2148			ctx->va_entrypoint,
2149			&attr, 1);
2150		✗	if (vas != VA_STATUS_SUCCESS) {
2151		✗	ctx->max_frame_size = 0;
2152		✗	av_log(avctx, AV_LOG_ERROR, "Failed to query max frame size "
2153			"config attribute: %d (%s).\n", vas, vaErrorStr(vas));
2154		✗	return AVERROR_EXTERNAL;
2155			}
2156
2157		✗	if (attr.value == VA_ATTRIB_NOT_SUPPORTED) {
2158		✗	ctx->max_frame_size = 0;
2159		✗	av_log(avctx, AV_LOG_ERROR, "Max frame size attribute "
2160			"is not supported.\n");
2161		✗	return AVERROR(EINVAL);
2162			} else {
2163			VAConfigAttribValMaxFrameSize attr_mfs;
2164		✗	attr_mfs.value = attr.value;
2165			// Prefer to use VAEncMiscParameterTypeMaxFrameSize for max frame size.
2166		✗	if (!attr_mfs.bits.max_frame_size && attr_mfs.bits.multiple_pass) {
2167		✗	ctx->max_frame_size = 0;
2168		✗	av_log(avctx, AV_LOG_ERROR, "Driver only supports multiple pass "
2169			"max frame size which has not been implemented in FFmpeg.\n");
2170		✗	return AVERROR(EINVAL);
2171			}
2172
2173		✗	ctx->mfs_params = (VAEncMiscParameterBufferMaxFrameSize){
2174		✗	.max_frame_size = ctx->max_frame_size * 8,
2175			};
2176
2177		✗	av_log(avctx, AV_LOG_VERBOSE, "Set max frame size: %d bytes.\n",
2178			ctx->max_frame_size);
2179			}
2180			#else
2181			av_log(avctx, AV_LOG_ERROR, "The max frame size option is not supported with "
2182			"this VAAPI version.\n");
2183			return AVERROR(EINVAL);
2184			#endif
2185
2186		✗	return 0;
2187			}
2188
2189		✗	static av_cold int vaapi_encode_init_gop_structure(AVCodecContext *avctx)
2190			{
2191		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2192			VAStatus vas;
2193		✗	VAConfigAttrib attr = { VAConfigAttribEncMaxRefFrames };
2194			uint32_t ref_l0, ref_l1;
2195			int prediction_pre_only;
2196
2197		✗	vas = vaGetConfigAttributes(ctx->hwctx->display,
2198			ctx->va_profile,
2199			ctx->va_entrypoint,
2200			&attr, 1);
2201		✗	if (vas != VA_STATUS_SUCCESS) {
2202		✗	av_log(avctx, AV_LOG_ERROR, "Failed to query reference frames "
2203			"attribute: %d (%s).\n", vas, vaErrorStr(vas));
2204		✗	return AVERROR_EXTERNAL;
2205			}
2206
2207		✗	if (attr.value == VA_ATTRIB_NOT_SUPPORTED) {
2208		✗	ref_l0 = ref_l1 = 0;
2209			} else {
2210		✗	ref_l0 = attr.value & 0xffff;
2211		✗	ref_l1 = attr.value >> 16 & 0xffff;
2212			}
2213
2214		✗	ctx->p_to_gpb = 0;
2215		✗	prediction_pre_only = 0;
2216
2217			#if VA_CHECK_VERSION(1, 9, 0)
2218		✗	if (!(ctx->codec->flags & FLAG_INTRA_ONLY ||
2219		✗	avctx->gop_size <= 1)) {
2220		✗	attr = (VAConfigAttrib) { VAConfigAttribPredictionDirection };
2221		✗	vas = vaGetConfigAttributes(ctx->hwctx->display,
2222			ctx->va_profile,
2223			ctx->va_entrypoint,
2224			&attr, 1);
2225		✗	if (vas != VA_STATUS_SUCCESS) {
2226		✗	av_log(avctx, AV_LOG_WARNING, "Failed to query prediction direction "
2227			"attribute: %d (%s).\n", vas, vaErrorStr(vas));
2228		✗	return AVERROR_EXTERNAL;
2229		✗	} else if (attr.value == VA_ATTRIB_NOT_SUPPORTED) {
2230		✗	av_log(avctx, AV_LOG_VERBOSE, "Driver does not report any additional "
2231			"prediction constraints.\n");
2232			} else {
2233		✗	if (((ref_l0 > 0 || ref_l1 > 0) && !(attr.value & VA_PREDICTION_DIRECTION_PREVIOUS)) ||
2234		✗	((ref_l1 == 0) && (attr.value & (VA_PREDICTION_DIRECTION_FUTURE | VA_PREDICTION_DIRECTION_BI_NOT_EMPTY)))) {
2235		✗	av_log(avctx, AV_LOG_ERROR, "Driver report incorrect prediction "
2236			"direction attribute.\n");
2237		✗	return AVERROR_EXTERNAL;
2238			}
2239
2240		✗	if (!(attr.value & VA_PREDICTION_DIRECTION_FUTURE)) {
2241		✗	if (ref_l0 > 0 && ref_l1 > 0) {
2242		✗	prediction_pre_only = 1;
2243		✗	av_log(avctx, AV_LOG_VERBOSE, "Driver only support same reference "
2244			"lists for B-frames.\n");
2245			}
2246			}
2247
2248		✗	if (attr.value & VA_PREDICTION_DIRECTION_BI_NOT_EMPTY) {
2249		✗	if (ref_l0 > 0 && ref_l1 > 0) {
2250		✗	ctx->p_to_gpb = 1;
2251		✗	av_log(avctx, AV_LOG_VERBOSE, "Driver does not support P-frames, "
2252			"replacing them with B-frames.\n");
2253			}
2254			}
2255			}
2256			}
2257			#endif
2258
2259		✗	if (ctx->codec->flags & FLAG_INTRA_ONLY ||
2260		✗	avctx->gop_size <= 1) {
2261		✗	av_log(avctx, AV_LOG_VERBOSE, "Using intra frames only.\n");
2262		✗	ctx->gop_size = 1;
2263		✗	} else if (ref_l0 < 1) {
2264		✗	av_log(avctx, AV_LOG_ERROR, "Driver does not support any "
2265			"reference frames.\n");
2266		✗	return AVERROR(EINVAL);
2267		✗	} else if (!(ctx->codec->flags & FLAG_B_PICTURES) ||
2268		✗	ref_l1 < 1 || avctx->max_b_frames < 1 ||
2269			prediction_pre_only) {
2270		✗	if (ctx->p_to_gpb)
2271		✗	av_log(avctx, AV_LOG_VERBOSE, "Using intra and B-frames "
2272			"(supported references: %d / %d).\n",
2273			ref_l0, ref_l1);
2274			else
2275		✗	av_log(avctx, AV_LOG_VERBOSE, "Using intra and P-frames "
2276			"(supported references: %d / %d).\n", ref_l0, ref_l1);
2277		✗	ctx->gop_size = avctx->gop_size;
2278		✗	ctx->p_per_i = INT_MAX;
2279		✗	ctx->b_per_p = 0;
2280			} else {
2281		✗	if (ctx->p_to_gpb)
2282		✗	av_log(avctx, AV_LOG_VERBOSE, "Using intra and B-frames "
2283			"(supported references: %d / %d).\n",
2284			ref_l0, ref_l1);
2285			else
2286		✗	av_log(avctx, AV_LOG_VERBOSE, "Using intra, P- and B-frames "
2287			"(supported references: %d / %d).\n", ref_l0, ref_l1);
2288		✗	ctx->gop_size = avctx->gop_size;
2289		✗	ctx->p_per_i = INT_MAX;
2290		✗	ctx->b_per_p = avctx->max_b_frames;
2291		✗	if (ctx->codec->flags & FLAG_B_PICTURE_REFERENCES) {
2292		✗	ctx->max_b_depth = FFMIN(ctx->desired_b_depth,
2293			av_log2(ctx->b_per_p) + 1);
2294			} else {
2295		✗	ctx->max_b_depth = 1;
2296			}
2297			}
2298
2299		✗	if (ctx->codec->flags & FLAG_NON_IDR_KEY_PICTURES) {
2300		✗	ctx->closed_gop = !!(avctx->flags & AV_CODEC_FLAG_CLOSED_GOP);
2301		✗	ctx->gop_per_idr = ctx->idr_interval + 1;
2302			} else {
2303		✗	ctx->closed_gop = 1;
2304		✗	ctx->gop_per_idr = 1;
2305			}
2306
2307		✗	return 0;
2308			}
2309
2310		✗	static av_cold int vaapi_encode_init_row_slice_structure(AVCodecContext *avctx,
2311			uint32_t slice_structure)
2312			{
2313		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2314			int req_slices;
2315
2316			// For fixed-size slices currently we only support whole rows, making
2317			// rectangular slices. This could be extended to arbitrary runs of
2318			// blocks, but since slices tend to be a conformance requirement and
2319			// most cases (such as broadcast or bluray) want rectangular slices
2320			// only it would need to be gated behind another option.
2321		✗	if (avctx->slices > ctx->slice_block_rows) {
2322		✗	av_log(avctx, AV_LOG_WARNING, "Not enough rows to use "
2323			"configured number of slices (%d < %d); using "
2324			"maximum.\n", ctx->slice_block_rows, avctx->slices);
2325		✗	req_slices = ctx->slice_block_rows;
2326			} else {
2327		✗	req_slices = avctx->slices;
2328			}
2329		✗	if (slice_structure & VA_ENC_SLICE_STRUCTURE_ARBITRARY_ROWS ||
2330		✗	slice_structure & VA_ENC_SLICE_STRUCTURE_ARBITRARY_MACROBLOCKS) {
2331		✗	ctx->nb_slices = req_slices;
2332		✗	ctx->slice_size = ctx->slice_block_rows / ctx->nb_slices;
2333		✗	} else if (slice_structure & VA_ENC_SLICE_STRUCTURE_POWER_OF_TWO_ROWS) {
2334			int k;
2335		✗	for (k = 1;; k *= 2) {
2336		✗	if (2 * k * (req_slices - 1) + 1 >= ctx->slice_block_rows)
2337		✗	break;
2338			}
2339		✗	ctx->nb_slices = (ctx->slice_block_rows + k - 1) / k;
2340		✗	ctx->slice_size = k;
2341			#if VA_CHECK_VERSION(1, 0, 0)
2342		✗	} else if (slice_structure & VA_ENC_SLICE_STRUCTURE_EQUAL_ROWS) {
2343		✗	ctx->nb_slices = ctx->slice_block_rows;
2344		✗	ctx->slice_size = 1;
2345			#endif
2346			} else {
2347		✗	av_log(avctx, AV_LOG_ERROR, "Driver does not support any usable "
2348			"slice structure modes (%#x).\n", slice_structure);
2349		✗	return AVERROR(EINVAL);
2350			}
2351
2352		✗	return 0;
2353			}
2354
2355		✗	static av_cold int vaapi_encode_init_tile_slice_structure(AVCodecContext *avctx,
2356			uint32_t slice_structure)
2357			{
2358		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2359			int i, req_tiles;
2360
2361		✗	if (!(slice_structure & VA_ENC_SLICE_STRUCTURE_ARBITRARY_MACROBLOCKS ||
2362		✗	(slice_structure & VA_ENC_SLICE_STRUCTURE_ARBITRARY_ROWS &&
2363		✗	ctx->tile_cols == 1))) {
2364		✗	av_log(avctx, AV_LOG_ERROR, "Supported slice structure (%#x) doesn't work for "
2365			"current tile requirement.\n", slice_structure);
2366		✗	return AVERROR(EINVAL);
2367			}
2368
2369		✗	if (ctx->tile_rows > ctx->slice_block_rows ||
2370		✗	ctx->tile_cols > ctx->slice_block_cols) {
2371		✗	av_log(avctx, AV_LOG_WARNING, "Not enough block rows/cols (%d x %d) "
2372			"for configured number of tile (%d x %d); ",
2373			ctx->slice_block_rows, ctx->slice_block_cols,
2374			ctx->tile_rows, ctx->tile_cols);
2375		✗	ctx->tile_rows = ctx->tile_rows > ctx->slice_block_rows ?
2376		✗	ctx->slice_block_rows : ctx->tile_rows;
2377		✗	ctx->tile_cols = ctx->tile_cols > ctx->slice_block_cols ?
2378		✗	ctx->slice_block_cols : ctx->tile_cols;
2379		✗	av_log(avctx, AV_LOG_WARNING, "using allowed maximum (%d x %d).\n",
2380			ctx->tile_rows, ctx->tile_cols);
2381			}
2382
2383		✗	req_tiles = ctx->tile_rows * ctx->tile_cols;
2384
2385			// Tile slice is not allowed to cross the boundary of a tile due to
2386			// the constraints of media-driver. Currently we support one slice
2387			// per tile. This could be extended to multiple slices per tile.
2388		✗	if (avctx->slices != req_tiles)
2389		✗	av_log(avctx, AV_LOG_WARNING, "The number of requested slices "
2390			"mismatches with configured number of tile (%d != %d); "
2391			"using requested tile number for slice.\n",
2392			avctx->slices, req_tiles);
2393
2394		✗	ctx->nb_slices = req_tiles;
2395
2396			// Default in uniform spacing
2397			// 6-3, 6-5
2398		✗	for (i = 0; i < ctx->tile_cols; i++) {
2399		✗	ctx->col_width[i] = ( i + 1 ) * ctx->slice_block_cols / ctx->tile_cols -
2400		✗	i * ctx->slice_block_cols / ctx->tile_cols;
2401		✗	ctx->col_bd[i + 1] = ctx->col_bd[i] + ctx->col_width[i];
2402			}
2403			// 6-4, 6-6
2404		✗	for (i = 0; i < ctx->tile_rows; i++) {
2405		✗	ctx->row_height[i] = ( i + 1 ) * ctx->slice_block_rows / ctx->tile_rows -
2406		✗	i * ctx->slice_block_rows / ctx->tile_rows;
2407		✗	ctx->row_bd[i + 1] = ctx->row_bd[i] + ctx->row_height[i];
2408			}
2409
2410		✗	av_log(avctx, AV_LOG_VERBOSE, "Encoding pictures with %d x %d tile.\n",
2411			ctx->tile_rows, ctx->tile_cols);
2412
2413		✗	return 0;
2414			}
2415
2416		✗	static av_cold int vaapi_encode_init_slice_structure(AVCodecContext *avctx)
2417			{
2418		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2419		✗	VAConfigAttrib attr[3] = { { VAConfigAttribEncMaxSlices },
2420			{ VAConfigAttribEncSliceStructure },
2421			#if VA_CHECK_VERSION(1, 1, 0)
2422			{ VAConfigAttribEncTileSupport },
2423			#endif
2424			};
2425			VAStatus vas;
2426			uint32_t max_slices, slice_structure;
2427			int ret;
2428
2429		✗	if (!(ctx->codec->flags & FLAG_SLICE_CONTROL)) {
2430		✗	if (avctx->slices > 0) {
2431		✗	av_log(avctx, AV_LOG_WARNING, "Multiple slices were requested "
2432			"but this codec does not support controlling slices.\n");
2433			}
2434		✗	return 0;
2435			}
2436
2437		✗	av_assert0(ctx->slice_block_height > 0 && ctx->slice_block_width > 0);
2438
2439		✗	ctx->slice_block_rows = (avctx->height + ctx->slice_block_height - 1) /
2440		✗	ctx->slice_block_height;
2441		✗	ctx->slice_block_cols = (avctx->width + ctx->slice_block_width - 1) /
2442		✗	ctx->slice_block_width;
2443
2444		✗	if (avctx->slices <= 1 && !ctx->tile_rows && !ctx->tile_cols) {
2445		✗	ctx->nb_slices = 1;
2446		✗	ctx->slice_size = ctx->slice_block_rows;
2447		✗	return 0;
2448			}
2449
2450		✗	vas = vaGetConfigAttributes(ctx->hwctx->display,
2451			ctx->va_profile,
2452			ctx->va_entrypoint,
2453			attr, FF_ARRAY_ELEMS(attr));
2454		✗	if (vas != VA_STATUS_SUCCESS) {
2455		✗	av_log(avctx, AV_LOG_ERROR, "Failed to query slice "
2456			"attributes: %d (%s).\n", vas, vaErrorStr(vas));
2457		✗	return AVERROR_EXTERNAL;
2458			}
2459		✗	max_slices = attr[0].value;
2460		✗	slice_structure = attr[1].value;
2461		✗	if (max_slices == VA_ATTRIB_NOT_SUPPORTED ||
2462			slice_structure == VA_ATTRIB_NOT_SUPPORTED) {
2463		✗	av_log(avctx, AV_LOG_ERROR, "Driver does not support encoding "
2464			"pictures as multiple slices.\n.");
2465		✗	return AVERROR(EINVAL);
2466			}
2467
2468		✗	if (ctx->tile_rows && ctx->tile_cols) {
2469			#if VA_CHECK_VERSION(1, 1, 0)
2470		✗	uint32_t tile_support = attr[2].value;
2471		✗	if (tile_support == VA_ATTRIB_NOT_SUPPORTED) {
2472		✗	av_log(avctx, AV_LOG_ERROR, "Driver does not support encoding "
2473			"pictures as multiple tiles.\n.");
2474		✗	return AVERROR(EINVAL);
2475			}
2476			#else
2477			av_log(avctx, AV_LOG_ERROR, "Tile encoding option is "
2478			"not supported with this VAAPI version.\n");
2479			return AVERROR(EINVAL);
2480			#endif
2481			}
2482
2483		✗	if (ctx->tile_rows && ctx->tile_cols)
2484		✗	ret = vaapi_encode_init_tile_slice_structure(avctx, slice_structure);
2485			else
2486		✗	ret = vaapi_encode_init_row_slice_structure(avctx, slice_structure);
2487		✗	if (ret < 0)
2488		✗	return ret;
2489
2490		✗	if (ctx->nb_slices > avctx->slices) {
2491		✗	av_log(avctx, AV_LOG_WARNING, "Slice count rounded up to "
2492			"%d (from %d) due to driver constraints on slice "
2493			"structure.\n", ctx->nb_slices, avctx->slices);
2494			}
2495		✗	if (ctx->nb_slices > max_slices) {
2496		✗	av_log(avctx, AV_LOG_ERROR, "Driver does not support "
2497			"encoding with %d slices (max %"PRIu32").\n",
2498			ctx->nb_slices, max_slices);
2499		✗	return AVERROR(EINVAL);
2500			}
2501
2502		✗	av_log(avctx, AV_LOG_VERBOSE, "Encoding pictures with %d slices.\n",
2503			ctx->nb_slices);
2504		✗	return 0;
2505			}
2506
2507		✗	static av_cold int vaapi_encode_init_packed_headers(AVCodecContext *avctx)
2508			{
2509		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2510			VAStatus vas;
2511		✗	VAConfigAttrib attr = { VAConfigAttribEncPackedHeaders };
2512
2513		✗	vas = vaGetConfigAttributes(ctx->hwctx->display,
2514			ctx->va_profile,
2515			ctx->va_entrypoint,
2516			&attr, 1);
2517		✗	if (vas != VA_STATUS_SUCCESS) {
2518		✗	av_log(avctx, AV_LOG_ERROR, "Failed to query packed headers "
2519			"attribute: %d (%s).\n", vas, vaErrorStr(vas));
2520		✗	return AVERROR_EXTERNAL;
2521			}
2522
2523		✗	if (attr.value == VA_ATTRIB_NOT_SUPPORTED) {
2524		✗	if (ctx->desired_packed_headers) {
2525		✗	av_log(avctx, AV_LOG_WARNING, "Driver does not support any "
2526			"packed headers (wanted %#x).\n",
2527			ctx->desired_packed_headers);
2528			} else {
2529		✗	av_log(avctx, AV_LOG_VERBOSE, "Driver does not support any "
2530			"packed headers (none wanted).\n");
2531			}
2532		✗	ctx->va_packed_headers = 0;
2533			} else {
2534		✗	if (ctx->desired_packed_headers & ~attr.value) {
2535		✗	av_log(avctx, AV_LOG_WARNING, "Driver does not support some "
2536			"wanted packed headers (wanted %#x, found %#x).\n",
2537			ctx->desired_packed_headers, attr.value);
2538			} else {
2539		✗	av_log(avctx, AV_LOG_VERBOSE, "All wanted packed headers "
2540			"available (wanted %#x, found %#x).\n",
2541			ctx->desired_packed_headers, attr.value);
2542			}
2543		✗	ctx->va_packed_headers = ctx->desired_packed_headers & attr.value;
2544			}
2545
2546		✗	if (ctx->va_packed_headers) {
2547		✗	ctx->config_attributes[ctx->nb_config_attributes++] =
2548			(VAConfigAttrib) {
2549			.type = VAConfigAttribEncPackedHeaders,
2550		✗	.value = ctx->va_packed_headers,
2551			};
2552			}
2553
2554		✗	if ( (ctx->desired_packed_headers & VA_ENC_PACKED_HEADER_SEQUENCE) &&
2555		✗	!(ctx->va_packed_headers & VA_ENC_PACKED_HEADER_SEQUENCE) &&
2556		✗	(avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER)) {
2557		✗	av_log(avctx, AV_LOG_WARNING, "Driver does not support packed "
2558			"sequence headers, but a global header is requested.\n");
2559		✗	av_log(avctx, AV_LOG_WARNING, "No global header will be written: "
2560			"this may result in a stream which is not usable for some "
2561			"purposes (e.g. not muxable to some containers).\n");
2562			}
2563
2564		✗	return 0;
2565			}
2566
2567		✗	static av_cold int vaapi_encode_init_quality(AVCodecContext *avctx)
2568			{
2569			#if VA_CHECK_VERSION(0, 36, 0)
2570		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2571			VAStatus vas;
2572		✗	VAConfigAttrib attr = { VAConfigAttribEncQualityRange };
2573		✗	int quality = avctx->compression_level;
2574
2575		✗	vas = vaGetConfigAttributes(ctx->hwctx->display,
2576			ctx->va_profile,
2577			ctx->va_entrypoint,
2578			&attr, 1);
2579		✗	if (vas != VA_STATUS_SUCCESS) {
2580		✗	av_log(avctx, AV_LOG_ERROR, "Failed to query quality "
2581			"config attribute: %d (%s).\n", vas, vaErrorStr(vas));
2582		✗	return AVERROR_EXTERNAL;
2583			}
2584
2585		✗	if (attr.value == VA_ATTRIB_NOT_SUPPORTED) {
2586		✗	if (quality != 0) {
2587		✗	av_log(avctx, AV_LOG_WARNING, "Quality attribute is not "
2588			"supported: will use default quality level.\n");
2589			}
2590			} else {
2591		✗	if (quality > attr.value) {
2592		✗	av_log(avctx, AV_LOG_WARNING, "Invalid quality level: "
2593			"valid range is 0-%d, using %d.\n",
2594			attr.value, attr.value);
2595		✗	quality = attr.value;
2596			}
2597
2598		✗	ctx->quality_params = (VAEncMiscParameterBufferQualityLevel) {
2599			.quality_level = quality,
2600			};
2601		✗	vaapi_encode_add_global_param(avctx,
2602			VAEncMiscParameterTypeQualityLevel,
2603		✗	&ctx->quality_params,
2604			sizeof(ctx->quality_params));
2605			}
2606			#else
2607			av_log(avctx, AV_LOG_WARNING, "The encode quality option is "
2608			"not supported with this VAAPI version.\n");
2609			#endif
2610
2611		✗	return 0;
2612			}
2613
2614		✗	static av_cold int vaapi_encode_init_roi(AVCodecContext *avctx)
2615			{
2616			#if VA_CHECK_VERSION(1, 0, 0)
2617		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2618			VAStatus vas;
2619		✗	VAConfigAttrib attr = { VAConfigAttribEncROI };
2620
2621		✗	vas = vaGetConfigAttributes(ctx->hwctx->display,
2622			ctx->va_profile,
2623			ctx->va_entrypoint,
2624			&attr, 1);
2625		✗	if (vas != VA_STATUS_SUCCESS) {
2626		✗	av_log(avctx, AV_LOG_ERROR, "Failed to query ROI "
2627			"config attribute: %d (%s).\n", vas, vaErrorStr(vas));
2628		✗	return AVERROR_EXTERNAL;
2629			}
2630
2631		✗	if (attr.value == VA_ATTRIB_NOT_SUPPORTED) {
2632		✗	ctx->roi_allowed = 0;
2633			} else {
2634		✗	VAConfigAttribValEncROI roi = {
2635		✗	.value = attr.value,
2636			};
2637
2638		✗	ctx->roi_max_regions = roi.bits.num_roi_regions;
2639		✗	ctx->roi_allowed = ctx->roi_max_regions > 0 &&
2640		✗	(ctx->va_rc_mode == VA_RC_CQP ||
2641			roi.bits.roi_rc_qp_delta_support);
2642			}
2643			#endif
2644		✗	return 0;
2645			}
2646
2647		✗	static void vaapi_encode_free_output_buffer(FFRefStructOpaque opaque,
2648			void *obj)
2649			{
2650		✗	AVCodecContext *avctx = opaque.nc;
2651		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2652		✗	VABufferID *buffer_id_ref = obj;
2653		✗	VABufferID buffer_id = *buffer_id_ref;
2654
2655		✗	vaDestroyBuffer(ctx->hwctx->display, buffer_id);
2656
2657		✗	av_log(avctx, AV_LOG_DEBUG, "Freed output buffer %#x\n", buffer_id);
2658		✗	}
2659
2660		✗	static int vaapi_encode_alloc_output_buffer(FFRefStructOpaque opaque, void *obj)
2661			{
2662		✗	AVCodecContext *avctx = opaque.nc;
2663		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2664		✗	VABufferID *buffer_id = obj;
2665			VAStatus vas;
2666
2667			// The output buffer size is fixed, so it needs to be large enough
2668			// to hold the largest possible compressed frame. We assume here
2669			// that the uncompressed frame plus some header data is an upper
2670			// bound on that.
2671		✗	vas = vaCreateBuffer(ctx->hwctx->display, ctx->va_context,
2672			VAEncCodedBufferType,
2673		✗	3 * ctx->surface_width * ctx->surface_height +
2674			(1 << 16), 1, 0, buffer_id);
2675		✗	if (vas != VA_STATUS_SUCCESS) {
2676		✗	av_log(avctx, AV_LOG_ERROR, "Failed to create bitstream "
2677			"output buffer: %d (%s).\n", vas, vaErrorStr(vas));
2678		✗	return AVERROR(ENOMEM);
2679			}
2680
2681		✗	av_log(avctx, AV_LOG_DEBUG, "Allocated output buffer %#x\n", *buffer_id);
2682
2683		✗	return 0;
2684			}
2685
2686		✗	static av_cold int vaapi_encode_create_recon_frames(AVCodecContext *avctx)
2687			{
2688		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2689		✗	AVVAAPIHWConfig *hwconfig = NULL;
2690		✗	AVHWFramesConstraints *constraints = NULL;
2691			enum AVPixelFormat recon_format;
2692			int err, i;
2693
2694		✗	hwconfig = av_hwdevice_hwconfig_alloc(ctx->device_ref);
2695		✗	if (!hwconfig) {
2696		✗	err = AVERROR(ENOMEM);
2697		✗	goto fail;
2698			}
2699		✗	hwconfig->config_id = ctx->va_config;
2700
2701		✗	constraints = av_hwdevice_get_hwframe_constraints(ctx->device_ref,
2702			hwconfig);
2703		✗	if (!constraints) {
2704		✗	err = AVERROR(ENOMEM);
2705		✗	goto fail;
2706			}
2707
2708			// Probably we can use the input surface format as the surface format
2709			// of the reconstructed frames. If not, we just pick the first (only?)
2710			// format in the valid list and hope that it all works.
2711		✗	recon_format = AV_PIX_FMT_NONE;
2712		✗	if (constraints->valid_sw_formats) {
2713		✗	for (i = 0; constraints->valid_sw_formats[i] != AV_PIX_FMT_NONE; i++) {
2714		✗	if (ctx->input_frames->sw_format ==
2715		✗	constraints->valid_sw_formats[i]) {
2716		✗	recon_format = ctx->input_frames->sw_format;
2717		✗	break;
2718			}
2719			}
2720		✗	if (recon_format == AV_PIX_FMT_NONE) {
2721			// No match. Just use the first in the supported list and
2722			// hope for the best.
2723		✗	recon_format = constraints->valid_sw_formats[0];
2724			}
2725			} else {
2726			// No idea what to use; copy input format.
2727		✗	recon_format = ctx->input_frames->sw_format;
2728			}
2729		✗	av_log(avctx, AV_LOG_DEBUG, "Using %s as format of "
2730			"reconstructed frames.\n", av_get_pix_fmt_name(recon_format));
2731
2732		✗	if (ctx->surface_width < constraints->min_width ||
2733		✗	ctx->surface_height < constraints->min_height ||
2734		✗	ctx->surface_width > constraints->max_width ||
2735		✗	ctx->surface_height > constraints->max_height) {
2736		✗	av_log(avctx, AV_LOG_ERROR, "Hardware does not support encoding at "
2737			"size %dx%d (constraints: width %d-%d height %d-%d).\n",
2738			ctx->surface_width, ctx->surface_height,
2739		✗	constraints->min_width, constraints->max_width,
2740		✗	constraints->min_height, constraints->max_height);
2741		✗	err = AVERROR(EINVAL);
2742		✗	goto fail;
2743			}
2744
2745		✗	av_freep(&hwconfig);
2746		✗	av_hwframe_constraints_free(&constraints);
2747
2748		✗	ctx->recon_frames_ref = av_hwframe_ctx_alloc(ctx->device_ref);
2749		✗	if (!ctx->recon_frames_ref) {
2750		✗	err = AVERROR(ENOMEM);
2751		✗	goto fail;
2752			}
2753		✗	ctx->recon_frames = (AVHWFramesContext*)ctx->recon_frames_ref->data;
2754
2755		✗	ctx->recon_frames->format = AV_PIX_FMT_VAAPI;
2756		✗	ctx->recon_frames->sw_format = recon_format;
2757		✗	ctx->recon_frames->width = ctx->surface_width;
2758		✗	ctx->recon_frames->height = ctx->surface_height;
2759
2760		✗	err = av_hwframe_ctx_init(ctx->recon_frames_ref);
2761		✗	if (err < 0) {
2762		✗	av_log(avctx, AV_LOG_ERROR, "Failed to initialise reconstructed "
2763			"frame context: %d.\n", err);
2764		✗	goto fail;
2765			}
2766
2767		✗	err = 0;
2768		✗	fail:
2769		✗	av_freep(&hwconfig);
2770		✗	av_hwframe_constraints_free(&constraints);
2771		✗	return err;
2772			}
2773
2774		✗	av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
2775			{
2776		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2777		✗	AVVAAPIFramesContext *recon_hwctx = NULL;
2778			VAStatus vas;
2779			int err;
2780
2781		✗	ctx->va_config = VA_INVALID_ID;
2782		✗	ctx->va_context = VA_INVALID_ID;
2783
2784			/* If you add something that can fail above this av_frame_alloc(),
2785			* modify ff_vaapi_encode_close() accordingly. */
2786		✗	ctx->frame = av_frame_alloc();
2787		✗	if (!ctx->frame) {
2788		✗	return AVERROR(ENOMEM);
2789			}
2790
2791		✗	if (!avctx->hw_frames_ctx) {
2792		✗	av_log(avctx, AV_LOG_ERROR, "A hardware frames reference is "
2793			"required to associate the encoding device.\n");
2794		✗	return AVERROR(EINVAL);
2795			}
2796
2797		✗	ctx->input_frames_ref = av_buffer_ref(avctx->hw_frames_ctx);
2798		✗	if (!ctx->input_frames_ref) {
2799		✗	err = AVERROR(ENOMEM);
2800		✗	goto fail;
2801			}
2802		✗	ctx->input_frames = (AVHWFramesContext*)ctx->input_frames_ref->data;
2803
2804		✗	ctx->device_ref = av_buffer_ref(ctx->input_frames->device_ref);
2805		✗	if (!ctx->device_ref) {
2806		✗	err = AVERROR(ENOMEM);
2807		✗	goto fail;
2808			}
2809		✗	ctx->device = (AVHWDeviceContext*)ctx->device_ref->data;
2810		✗	ctx->hwctx = ctx->device->hwctx;
2811
2812		✗	ctx->tail_pkt = av_packet_alloc();
2813		✗	if (!ctx->tail_pkt) {
2814		✗	err = AVERROR(ENOMEM);
2815		✗	goto fail;
2816			}
2817
2818		✗	err = vaapi_encode_profile_entrypoint(avctx);
2819		✗	if (err < 0)
2820		✗	goto fail;
2821
2822		✗	if (ctx->codec->get_encoder_caps) {
2823		✗	err = ctx->codec->get_encoder_caps(avctx);
2824		✗	if (err < 0)
2825		✗	goto fail;
2826			} else {
2827			// Assume 16x16 blocks.
2828		✗	ctx->surface_width = FFALIGN(avctx->width, 16);
2829		✗	ctx->surface_height = FFALIGN(avctx->height, 16);
2830		✗	if (ctx->codec->flags & FLAG_SLICE_CONTROL) {
2831		✗	ctx->slice_block_width = 16;
2832		✗	ctx->slice_block_height = 16;
2833			}
2834			}
2835
2836		✗	err = vaapi_encode_init_rate_control(avctx);
2837		✗	if (err < 0)
2838		✗	goto fail;
2839
2840		✗	err = vaapi_encode_init_gop_structure(avctx);
2841		✗	if (err < 0)
2842		✗	goto fail;
2843
2844		✗	err = vaapi_encode_init_slice_structure(avctx);
2845		✗	if (err < 0)
2846		✗	goto fail;
2847
2848		✗	err = vaapi_encode_init_packed_headers(avctx);
2849		✗	if (err < 0)
2850		✗	goto fail;
2851
2852		✗	err = vaapi_encode_init_roi(avctx);
2853		✗	if (err < 0)
2854		✗	goto fail;
2855
2856		✗	if (avctx->compression_level >= 0) {
2857		✗	err = vaapi_encode_init_quality(avctx);
2858		✗	if (err < 0)
2859		✗	goto fail;
2860			}
2861
2862		✗	if (ctx->max_frame_size) {
2863		✗	err = vaapi_encode_init_max_frame_size(avctx);
2864		✗	if (err < 0)
2865		✗	goto fail;
2866			}
2867
2868		✗	vas = vaCreateConfig(ctx->hwctx->display,
2869			ctx->va_profile, ctx->va_entrypoint,
2870		✗	ctx->config_attributes, ctx->nb_config_attributes,
2871			&ctx->va_config);
2872		✗	if (vas != VA_STATUS_SUCCESS) {
2873		✗	av_log(avctx, AV_LOG_ERROR, "Failed to create encode pipeline "
2874			"configuration: %d (%s).\n", vas, vaErrorStr(vas));
2875		✗	err = AVERROR(EIO);
2876		✗	goto fail;
2877			}
2878
2879		✗	err = vaapi_encode_create_recon_frames(avctx);
2880		✗	if (err < 0)
2881		✗	goto fail;
2882
2883		✗	recon_hwctx = ctx->recon_frames->hwctx;
2884		✗	vas = vaCreateContext(ctx->hwctx->display, ctx->va_config,
2885			ctx->surface_width, ctx->surface_height,
2886			VA_PROGRESSIVE,
2887			recon_hwctx->surface_ids,
2888			recon_hwctx->nb_surfaces,
2889			&ctx->va_context);
2890		✗	if (vas != VA_STATUS_SUCCESS) {
2891		✗	av_log(avctx, AV_LOG_ERROR, "Failed to create encode pipeline "
2892			"context: %d (%s).\n", vas, vaErrorStr(vas));
2893		✗	err = AVERROR(EIO);
2894		✗	goto fail;
2895			}
2896
2897		✗	ctx->output_buffer_pool =
2898		✗	ff_refstruct_pool_alloc_ext(sizeof(VABufferID), 0, avctx,
2899			&vaapi_encode_alloc_output_buffer, NULL,
2900			vaapi_encode_free_output_buffer, NULL);
2901		✗	if (!ctx->output_buffer_pool) {
2902		✗	err = AVERROR(ENOMEM);
2903		✗	goto fail;
2904			}
2905
2906		✗	if (ctx->codec->configure) {
2907		✗	err = ctx->codec->configure(avctx);
2908		✗	if (err < 0)
2909		✗	goto fail;
2910			}
2911
2912		✗	ctx->output_delay = ctx->b_per_p;
2913		✗	ctx->decode_delay = ctx->max_b_depth;
2914
2915		✗	if (ctx->codec->sequence_params_size > 0) {
2916		✗	ctx->codec_sequence_params =
2917		✗	av_mallocz(ctx->codec->sequence_params_size);
2918		✗	if (!ctx->codec_sequence_params) {
2919		✗	err = AVERROR(ENOMEM);
2920		✗	goto fail;
2921			}
2922			}
2923		✗	if (ctx->codec->picture_params_size > 0) {
2924		✗	ctx->codec_picture_params =
2925		✗	av_mallocz(ctx->codec->picture_params_size);
2926		✗	if (!ctx->codec_picture_params) {
2927		✗	err = AVERROR(ENOMEM);
2928		✗	goto fail;
2929			}
2930			}
2931
2932		✗	if (ctx->codec->init_sequence_params) {
2933		✗	err = ctx->codec->init_sequence_params(avctx);
2934		✗	if (err < 0) {
2935		✗	av_log(avctx, AV_LOG_ERROR, "Codec sequence initialisation "
2936			"failed: %d.\n", err);
2937		✗	goto fail;
2938			}
2939			}
2940
2941		✗	if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_SEQUENCE &&
2942		✗	ctx->codec->write_sequence_header &&
2943		✗	avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) {
2944			char data[MAX_PARAM_BUFFER_SIZE];
2945		✗	size_t bit_len = 8 * sizeof(data);
2946
2947		✗	err = ctx->codec->write_sequence_header(avctx, data, &bit_len);
2948		✗	if (err < 0) {
2949		✗	av_log(avctx, AV_LOG_ERROR, "Failed to write sequence header "
2950			"for extradata: %d.\n", err);
2951		✗	goto fail;
2952			} else {
2953		✗	avctx->extradata_size = (bit_len + 7) / 8;
2954		✗	avctx->extradata = av_mallocz(avctx->extradata_size +
2955			AV_INPUT_BUFFER_PADDING_SIZE);
2956		✗	if (!avctx->extradata) {
2957		✗	err = AVERROR(ENOMEM);
2958		✗	goto fail;
2959			}
2960		✗	memcpy(avctx->extradata, data, avctx->extradata_size);
2961			}
2962			}
2963
2964			#if VA_CHECK_VERSION(1, 9, 0)
2965			// check vaSyncBuffer function
2966		✗	vas = vaSyncBuffer(ctx->hwctx->display, VA_INVALID_ID, 0);
2967		✗	if (vas != VA_STATUS_ERROR_UNIMPLEMENTED) {
2968		✗	ctx->has_sync_buffer_func = 1;
2969		✗	ctx->encode_fifo = av_fifo_alloc2(ctx->async_depth,
2970			sizeof(VAAPIEncodePicture *),
2971			0);
2972		✗	if (!ctx->encode_fifo)
2973		✗	return AVERROR(ENOMEM);
2974			}
2975			#endif
2976
2977		✗	return 0;
2978
2979		✗	fail:
2980		✗	return err;
2981			}
2982
2983		✗	av_cold int ff_vaapi_encode_close(AVCodecContext *avctx)
2984			{
2985		✗	VAAPIEncodeContext *ctx = avctx->priv_data;
2986			VAAPIEncodePicture *pic, *next;
2987
2988			/* We check ctx->frame to know whether ff_vaapi_encode_init()
2989			* has been called and va_config/va_context initialized. */
2990		✗	if (!ctx->frame)
2991		✗	return 0;
2992
2993		✗	for (pic = ctx->pic_start; pic; pic = next) {
2994		✗	next = pic->next;
2995		✗	vaapi_encode_free(avctx, pic);
2996			}
2997
2998		✗	ff_refstruct_pool_uninit(&ctx->output_buffer_pool);
2999
3000		✗	if (ctx->va_context != VA_INVALID_ID) {
3001		✗	vaDestroyContext(ctx->hwctx->display, ctx->va_context);
3002		✗	ctx->va_context = VA_INVALID_ID;
3003			}
3004
3005		✗	if (ctx->va_config != VA_INVALID_ID) {
3006		✗	vaDestroyConfig(ctx->hwctx->display, ctx->va_config);
3007		✗	ctx->va_config = VA_INVALID_ID;
3008			}
3009
3010		✗	av_frame_free(&ctx->frame);
3011		✗	av_packet_free(&ctx->tail_pkt);
3012
3013		✗	av_freep(&ctx->codec_sequence_params);
3014		✗	av_freep(&ctx->codec_picture_params);
3015		✗	av_fifo_freep2(&ctx->encode_fifo);
3016
3017		✗	av_buffer_unref(&ctx->recon_frames_ref);
3018		✗	av_buffer_unref(&ctx->input_frames_ref);
3019		✗	av_buffer_unref(&ctx->device_ref);
3020
3021		✗	return 0;
3022			}
3023