FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* This file is part of FFmpeg.
3			*
4			* FFmpeg is free software; you can redistribute it and/or
5			* modify it under the terms of the GNU Lesser General Public
6			* License as published by the Free Software Foundation; either
7			* version 2.1 of the License, or (at your option) any later version.
8			*
9			* FFmpeg is distributed in the hope that it will be useful,
10			* but WITHOUT ANY WARRANTY; without even the implied warranty of
11			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12			* Lesser General Public License for more details.
13			*
14			* You should have received a copy of the GNU Lesser General Public
15			* License along with FFmpeg; if not, write to the Free Software
16			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17			*/
18
19			#define VK_NO_PROTOTYPES
20			#define VK_ENABLE_BETA_EXTENSIONS
21
22			#include "config.h"
23			#include "ffplay_renderer.h"
24
25			#if (SDL_VERSION_ATLEAST(2, 0, 6) && CONFIG_LIBPLACEBO)
26			#define HAVE_VULKAN_RENDERER 1
27			#else
28			#define HAVE_VULKAN_RENDERER 0
29			#endif
30
31			#if HAVE_VULKAN_RENDERER
32
33			#if defined(_WIN32) && !defined(VK_USE_PLATFORM_WIN32_KHR)
34			#define VK_USE_PLATFORM_WIN32_KHR
35			#endif
36
37			#include <libplacebo/vulkan.h>
38			#include <libplacebo/utils/frame_queue.h>
39			#include <libplacebo/utils/libav.h>
40			#include <SDL_vulkan.h>
41
42			#include "libavutil/bprint.h"
43			#include "libavutil/mem.h"
44			#include "libavutil/internal.h"
45
46			#endif
47
48			struct VkRenderer {
49			const AVClass *class;
50
51			int (*create)(VkRenderer *renderer, SDL_Window *window, AVDictionary *dict);
52
53			int (*get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev);
54
55			int (*display)(VkRenderer *renderer, AVFrame *frame, RenderParams *params);
56
57			int (*resize)(VkRenderer *renderer, int width, int height);
58
59			void (*destroy)(VkRenderer *renderer);
60			};
61
62			#if HAVE_VULKAN_RENDERER
63
64			typedef struct RendererContext {
65			VkRenderer api;
66
67			// Can be NULL when vulkan instance is created by avutil
68			pl_vk_inst placebo_instance;
69			pl_vulkan placebo_vulkan;
70			pl_swapchain swapchain;
71			VkSurfaceKHR vk_surface;
72			pl_renderer renderer;
73			pl_tex tex[4];
74
75			pl_log vk_log;
76
77			AVBufferRef *hw_device_ref;
78			AVBufferRef *hw_frame_ref;
79			enum AVPixelFormat *transfer_formats;
80			AVHWFramesConstraints *constraints;
81
82			PFN_vkGetInstanceProcAddr get_proc_addr;
83			// This field is a copy from pl_vk_inst->instance or hw_device_ref instance.
84			VkInstance inst;
85
86			AVFrame *vk_frame;
87			} RendererContext;
88
89			static void vk_log_cb(void *log_priv, enum pl_log_level level,
90			const char *msg)
91			{
92			static const int level_map[] = {
93			AV_LOG_QUIET,
94			AV_LOG_FATAL,
95			AV_LOG_ERROR,
96			AV_LOG_WARNING,
97			AV_LOG_INFO,
98			AV_LOG_DEBUG,
99			AV_LOG_TRACE,
100			};
101
102			if (level > 0 && level < FF_ARRAY_ELEMS(level_map))
103			av_log(log_priv, level_map[level], "%s\n", msg);
104			}
105
106			static inline int enable_debug(const AVDictionary *opt)
107			{
108			AVDictionaryEntry *entry = av_dict_get(opt, "debug", NULL, 0);
109			int debug = entry && strtol(entry->value, NULL, 10);
110			return debug;
111			}
112
113			static void hwctx_lock_queue(void *priv, uint32_t qf, uint32_t qidx)
114			{
115			AVHWDeviceContext *avhwctx = priv;
116			const AVVulkanDeviceContext *hwctx = avhwctx->hwctx;
117			#if FF_API_VULKAN_SYNC_QUEUES
118			FF_DISABLE_DEPRECATION_WARNINGS
119			hwctx->lock_queue(avhwctx, qf, qidx);
120			FF_ENABLE_DEPRECATION_WARNINGS
121			#endif
122			}
123
124			static void hwctx_unlock_queue(void *priv, uint32_t qf, uint32_t qidx)
125			{
126			AVHWDeviceContext *avhwctx = priv;
127			const AVVulkanDeviceContext *hwctx = avhwctx->hwctx;
128			#if FF_API_VULKAN_SYNC_QUEUES
129			FF_DISABLE_DEPRECATION_WARNINGS
130			hwctx->unlock_queue(avhwctx, qf, qidx);
131			FF_ENABLE_DEPRECATION_WARNINGS
132			#endif
133			}
134
135			static int add_instance_extension(const char **ext, unsigned num_ext,
136			const AVDictionary *opt,
137			AVDictionary **dict)
138			{
139			const char *inst_ext_key = "instance_extensions";
140			AVDictionaryEntry *entry;
141			AVBPrint buf;
142			char *ext_list = NULL;
143			int ret;
144
145			av_bprint_init(&buf, 0, AV_BPRINT_SIZE_AUTOMATIC);
146			for (int i = 0; i < num_ext; i++) {
147			if (i)
148			av_bprintf(&buf, "+%s", ext[i]);
149			else
150			av_bprintf(&buf, "%s", ext[i]);
151			}
152
153			entry = av_dict_get(opt, inst_ext_key, NULL, 0);
154			if (entry && entry->value && entry->value[0]) {
155			if (num_ext)
156			av_bprintf(&buf, "+");
157			av_bprintf(&buf, "%s", entry->value);
158			}
159
160			ret = av_bprint_finalize(&buf, &ext_list);
161			if (ret < 0)
162			return ret;
163			return av_dict_set(dict, inst_ext_key, ext_list, AV_DICT_DONT_STRDUP_VAL);
164			}
165
166			static int add_device_extension(const AVDictionary *opt,
167			AVDictionary **dict)
168			{
169			const char *dev_ext_key = "device_extensions";
170			AVDictionaryEntry *entry;
171			AVBPrint buf;
172			char *ext_list = NULL;
173			int ret;
174
175			av_bprint_init(&buf, 0, AV_BPRINT_SIZE_AUTOMATIC);
176			av_bprintf(&buf, "%s", VK_KHR_SWAPCHAIN_EXTENSION_NAME);
177			for (int i = 0; i < pl_vulkan_num_recommended_extensions; i++)
178			av_bprintf(&buf, "+%s", pl_vulkan_recommended_extensions[i]);
179
180			entry = av_dict_get(opt, dev_ext_key, NULL, 0);
181			if (entry && entry->value && entry->value[0])
182			av_bprintf(&buf, "+%s", entry->value);
183
184			ret = av_bprint_finalize(&buf, &ext_list);
185			if (ret < 0)
186			return ret;
187			return av_dict_set(dict, dev_ext_key, ext_list, AV_DICT_DONT_STRDUP_VAL);
188			}
189
190			static const char *select_device(const AVDictionary *opt)
191			{
192			const AVDictionaryEntry *entry;
193
194			entry = av_dict_get(opt, "device", NULL, 0);
195			if (entry)
196			return entry->value;
197			return NULL;
198			}
199
200			static int create_vk_by_hwcontext(VkRenderer *renderer,
201			const char **ext, unsigned num_ext,
202			const AVDictionary *opt)
203			{
204			RendererContext *ctx = (RendererContext *) renderer;
205			AVHWDeviceContext *dev;
206			AVVulkanDeviceContext *hwctx;
207			AVDictionary *dict = NULL;
208			int ret;
209
210			ret = add_instance_extension(ext, num_ext, opt, &dict);
211			if (ret < 0)
212			return ret;
213			ret = add_device_extension(opt, &dict);
214			if (ret) {
215			av_dict_free(&dict);
216			return ret;
217			}
218
219			ret = av_hwdevice_ctx_create(&ctx->hw_device_ref, AV_HWDEVICE_TYPE_VULKAN,
220			select_device(opt), dict, 0);
221			av_dict_free(&dict);
222			if (ret < 0)
223			return ret;
224
225			dev = (AVHWDeviceContext *) ctx->hw_device_ref->data;
226			hwctx = dev->hwctx;
227
228			// There is no way to pass SDL GetInstanceProcAddr to hwdevice.
229			// Check the result and return error if they don't match.
230			if (hwctx->get_proc_addr != SDL_Vulkan_GetVkGetInstanceProcAddr()) {
231			av_log(renderer, AV_LOG_ERROR,
232			"hwdevice and SDL use different get_proc_addr. "
233			"Try -vulkan_params create_by_placebo=1\n");
234			return AVERROR_PATCHWELCOME;
235			}
236
237			ctx->get_proc_addr = hwctx->get_proc_addr;
238			ctx->inst = hwctx->inst;
239
240			struct pl_vulkan_import_params import_params = {
241			.instance = hwctx->inst,
242			.get_proc_addr = hwctx->get_proc_addr,
243			.phys_device = hwctx->phys_dev,
244			.device = hwctx->act_dev,
245			.extensions = hwctx->enabled_dev_extensions,
246			.num_extensions = hwctx->nb_enabled_dev_extensions,
247			.features = &hwctx->device_features,
248			.lock_queue = hwctx_lock_queue,
249			.unlock_queue = hwctx_unlock_queue,
250			.queue_ctx = dev,
251			.queue_graphics = {
252			.index = VK_QUEUE_FAMILY_IGNORED,
253			.count = 0,
254			},
255			.queue_compute = {
256			.index = VK_QUEUE_FAMILY_IGNORED,
257			.count = 0,
258			},
259			.queue_transfer = {
260			.index = VK_QUEUE_FAMILY_IGNORED,
261			.count = 0,
262			},
263			};
264			for (int i = 0; i < hwctx->nb_qf; i++) {
265			const AVVulkanDeviceQueueFamily *qf = &hwctx->qf[i];
266
267			if (qf->flags & VK_QUEUE_GRAPHICS_BIT) {
268			import_params.queue_graphics.index = qf->idx;
269			import_params.queue_graphics.count = qf->num;
270			}
271			if (qf->flags & VK_QUEUE_COMPUTE_BIT) {
272			import_params.queue_compute.index = qf->idx;
273			import_params.queue_compute.count = qf->num;
274			}
275			if (qf->flags & VK_QUEUE_TRANSFER_BIT) {
276			import_params.queue_transfer.index = qf->idx;
277			import_params.queue_transfer.count = qf->num;
278			}
279			}
280
281			ctx->placebo_vulkan = pl_vulkan_import(ctx->vk_log, &import_params);
282			if (!ctx->placebo_vulkan)
283			return AVERROR_EXTERNAL;
284
285			return 0;
286			}
287
288			static void placebo_lock_queue(struct AVHWDeviceContext *dev_ctx,
289			uint32_t queue_family, uint32_t index)
290			{
291			RendererContext *ctx = dev_ctx->user_opaque;
292			pl_vulkan vk = ctx->placebo_vulkan;
293			#if FF_API_VULKAN_SYNC_QUEUES
294			FF_DISABLE_DEPRECATION_WARNINGS
295			vk->lock_queue(vk, queue_family, index);
296			FF_ENABLE_DEPRECATION_WARNINGS
297			#endif
298			}
299
300			static void placebo_unlock_queue(struct AVHWDeviceContext *dev_ctx,
301			uint32_t queue_family,
302			uint32_t index)
303			{
304			RendererContext *ctx = dev_ctx->user_opaque;
305			pl_vulkan vk = ctx->placebo_vulkan;
306			#if FF_API_VULKAN_SYNC_QUEUES
307			FF_DISABLE_DEPRECATION_WARNINGS
308			vk->unlock_queue(vk, queue_family, index);
309			FF_ENABLE_DEPRECATION_WARNINGS
310			#endif
311			}
312
313			static int get_decode_queue(VkRenderer *renderer, int *index, int *count)
314			{
315			RendererContext *ctx = (RendererContext *) renderer;
316			VkQueueFamilyProperties *queue_family_prop = NULL;
317			uint32_t num_queue_family_prop = 0;
318			PFN_vkGetPhysicalDeviceQueueFamilyProperties get_queue_family_prop;
319			PFN_vkGetInstanceProcAddr get_proc_addr = ctx->get_proc_addr;
320
321			*index = -1;
322			*count = 0;
323			get_queue_family_prop = (PFN_vkGetPhysicalDeviceQueueFamilyProperties)
324			get_proc_addr(ctx->placebo_instance->instance,
325			"vkGetPhysicalDeviceQueueFamilyProperties");
326			get_queue_family_prop(ctx->placebo_vulkan->phys_device,
327			&num_queue_family_prop, NULL);
328			if (!num_queue_family_prop)
329			return AVERROR_EXTERNAL;
330
331			queue_family_prop = av_calloc(num_queue_family_prop,
332			sizeof(*queue_family_prop));
333			if (!queue_family_prop)
334			return AVERROR(ENOMEM);
335
336			get_queue_family_prop(ctx->placebo_vulkan->phys_device,
337			&num_queue_family_prop,
338			queue_family_prop);
339
340			for (int i = 0; i < num_queue_family_prop; i++) {
341			if (queue_family_prop[i].queueFlags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) {
342			*index = i;
343			*count = queue_family_prop[i].queueCount;
344			break;
345			}
346			}
347			av_free(queue_family_prop);
348
349			return 0;
350			}
351
352			static int create_vk_by_placebo(VkRenderer *renderer,
353			const char **ext, unsigned num_ext,
354			const AVDictionary *opt)
355			{
356			RendererContext *ctx = (RendererContext *) renderer;
357			AVHWDeviceContext *device_ctx;
358			AVVulkanDeviceContext *vk_dev_ctx;
359			int decode_index;
360			int decode_count;
361			int ret;
362			const char **dev_exts;
363			int num_dev_exts;
364
365			ctx->get_proc_addr = SDL_Vulkan_GetVkGetInstanceProcAddr();
366
367			ctx->placebo_instance = pl_vk_inst_create(ctx->vk_log, pl_vk_inst_params(
368			.get_proc_addr = ctx->get_proc_addr,
369			.debug = enable_debug(opt),
370			.extensions = ext,
371			.num_extensions = num_ext
372			));
373			if (!ctx->placebo_instance) {
374			return AVERROR_EXTERNAL;
375			}
376			ctx->inst = ctx->placebo_instance->instance;
377
378			dev_exts = av_vk_get_optional_device_extensions(&num_dev_exts);
379			if (!dev_exts)
380			return AVERROR(ENOMEM);
381
382			ctx->placebo_vulkan = pl_vulkan_create(ctx->vk_log, pl_vulkan_params(
383			.instance = ctx->placebo_instance->instance,
384			.get_proc_addr = ctx->placebo_instance->get_proc_addr,
385			.surface = ctx->vk_surface,
386			.allow_software = false,
387			.opt_extensions = dev_exts,
388			.num_opt_extensions = num_dev_exts,
389			.extra_queues = VK_QUEUE_VIDEO_DECODE_BIT_KHR,
390			.device_name = select_device(opt),
391			));
392			av_free(dev_exts);
393			if (!ctx->placebo_vulkan)
394			return AVERROR_EXTERNAL;
395			ctx->hw_device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VULKAN);
396			if (!ctx->hw_device_ref) {
397			return AVERROR(ENOMEM);
398			}
399
400			device_ctx = (AVHWDeviceContext *) ctx->hw_device_ref->data;
401			device_ctx->user_opaque = ctx;
402
403			vk_dev_ctx = device_ctx->hwctx;
404			#if FF_API_VULKAN_SYNC_QUEUES
405			FF_DISABLE_DEPRECATION_WARNINGS
406			vk_dev_ctx->lock_queue = placebo_lock_queue;
407			vk_dev_ctx->unlock_queue = placebo_unlock_queue;
408			FF_ENABLE_DEPRECATION_WARNINGS
409			#endif
410
411			vk_dev_ctx->get_proc_addr = ctx->placebo_instance->get_proc_addr;
412
413			vk_dev_ctx->inst = ctx->placebo_instance->instance;
414			vk_dev_ctx->phys_dev = ctx->placebo_vulkan->phys_device;
415			vk_dev_ctx->act_dev = ctx->placebo_vulkan->device;
416
417			vk_dev_ctx->device_features = *ctx->placebo_vulkan->features;
418
419			vk_dev_ctx->enabled_inst_extensions = ctx->placebo_instance->extensions;
420			vk_dev_ctx->nb_enabled_inst_extensions = ctx->placebo_instance->num_extensions;
421
422			vk_dev_ctx->enabled_dev_extensions = ctx->placebo_vulkan->extensions;
423			vk_dev_ctx->nb_enabled_dev_extensions = ctx->placebo_vulkan->num_extensions;
424
425			int nb_qf = 0;
426			vk_dev_ctx->qf[nb_qf] = (AVVulkanDeviceQueueFamily) {
427			.idx = ctx->placebo_vulkan->queue_graphics.index,
428			.num = ctx->placebo_vulkan->queue_graphics.count,
429			.flags = VK_QUEUE_GRAPHICS_BIT,
430			};
431			nb_qf++;
432			vk_dev_ctx->qf[nb_qf] = (AVVulkanDeviceQueueFamily) {
433			.idx = ctx->placebo_vulkan->queue_transfer.index,
434			.num = ctx->placebo_vulkan->queue_transfer.count,
435			.flags = VK_QUEUE_TRANSFER_BIT,
436			};
437			nb_qf++;
438			vk_dev_ctx->qf[nb_qf] = (AVVulkanDeviceQueueFamily) {
439			.idx = ctx->placebo_vulkan->queue_compute.index,
440			.num = ctx->placebo_vulkan->queue_compute.count,
441			.flags = VK_QUEUE_COMPUTE_BIT,
442			};
443			nb_qf++;
444
445			ret = get_decode_queue(renderer, &decode_index, &decode_count);
446			if (ret < 0)
447			return ret;
448
449			vk_dev_ctx->qf[nb_qf] = (AVVulkanDeviceQueueFamily) {
450			.idx = decode_index,
451			.num = decode_count,
452			.flags = VK_QUEUE_VIDEO_DECODE_BIT_KHR,
453			};
454			nb_qf++;
455
456			vk_dev_ctx->nb_qf = nb_qf;
457
458			ret = av_hwdevice_ctx_init(ctx->hw_device_ref);
459			if (ret < 0)
460			return ret;
461
462			return 0;
463			}
464
465			static int create(VkRenderer *renderer, SDL_Window *window, AVDictionary *opt)
466			{
467			int ret = 0;
468			unsigned num_ext = 0;
469			const char **ext = NULL;
470			int w, h;
471			struct pl_log_params vk_log_params = {
472			.log_cb = vk_log_cb,
473			.log_level = PL_LOG_DEBUG,
474			.log_priv = renderer,
475			};
476			RendererContext *ctx = (RendererContext *) renderer;
477			AVDictionaryEntry *entry;
478
479			ctx->vk_log = pl_log_create(PL_API_VER, &vk_log_params);
480
481			if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) {
482			av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n",
483			SDL_GetError());
484			return AVERROR_EXTERNAL;
485			}
486
487			ext = av_calloc(num_ext, sizeof(*ext));
488			if (!ext) {
489			ret = AVERROR(ENOMEM);
490			goto out;
491			}
492
493			SDL_Vulkan_GetInstanceExtensions(window, &num_ext, ext);
494
495			entry = av_dict_get(opt, "create_by_placebo", NULL, 0);
496			if (entry && strtol(entry->value, NULL, 10))
497			ret = create_vk_by_placebo(renderer, ext, num_ext, opt);
498			else
499			ret = create_vk_by_hwcontext(renderer, ext, num_ext, opt);
500			if (ret < 0)
501			goto out;
502
503			if (!SDL_Vulkan_CreateSurface(window, ctx->inst, &ctx->vk_surface)) {
504			ret = AVERROR_EXTERNAL;
505			goto out;
506			}
507
508			ctx->swapchain = pl_vulkan_create_swapchain(
509			ctx->placebo_vulkan,
510			pl_vulkan_swapchain_params(
511			.surface = ctx->vk_surface,
512			.present_mode = VK_PRESENT_MODE_FIFO_KHR));
513			if (!ctx->swapchain) {
514			ret = AVERROR_EXTERNAL;
515			goto out;
516			}
517
518			SDL_Vulkan_GetDrawableSize(window, &w, &h);
519			pl_swapchain_resize(ctx->swapchain, &w, &h);
520
521			ctx->renderer = pl_renderer_create(ctx->vk_log, ctx->placebo_vulkan->gpu);
522			if (!ctx->renderer) {
523			ret = AVERROR_EXTERNAL;
524			goto out;
525			}
526
527			ctx->vk_frame = av_frame_alloc();
528			if (!ctx->vk_frame) {
529			ret = AVERROR(ENOMEM);
530			goto out;
531			}
532
533			ret = 0;
534
535			out:
536			av_free(ext);
537			return ret;
538			}
539
540			static int get_hw_dev(VkRenderer *renderer, AVBufferRef **dev)
541			{
542			RendererContext *ctx = (RendererContext *) renderer;
543
544			*dev = ctx->hw_device_ref;
545			return 0;
546			}
547
548			static int create_hw_frame(VkRenderer *renderer, AVFrame *frame)
549			{
550			RendererContext *ctx = (RendererContext *) renderer;
551			AVHWFramesContext *src_hw_frame = (AVHWFramesContext *)
552			frame->hw_frames_ctx->data;
553			AVHWFramesContext *hw_frame;
554			AVVulkanFramesContext *vk_frame_ctx;
555			int ret;
556
557			if (ctx->hw_frame_ref) {
558			hw_frame = (AVHWFramesContext *) ctx->hw_frame_ref->data;
559
560			if (hw_frame->width == frame->width &&
561			hw_frame->height == frame->height &&
562			hw_frame->sw_format == src_hw_frame->sw_format)
563			return 0;
564
565			av_buffer_unref(&ctx->hw_frame_ref);
566			}
567
568			if (!ctx->constraints) {
569			ctx->constraints = av_hwdevice_get_hwframe_constraints(
570			ctx->hw_device_ref, NULL);
571			if (!ctx->constraints)
572			return AVERROR(ENOMEM);
573			}
574
575			// Check constraints and skip create hwframe. Don't take it as error since
576			// we can fallback to memory copy from GPU to CPU.
577			if ((ctx->constraints->max_width &&
578			ctx->constraints->max_width < frame->width) ||
579			(ctx->constraints->max_height &&
580			ctx->constraints->max_height < frame->height) ||
581			(ctx->constraints->min_width &&
582			ctx->constraints->min_width > frame->width) ||
583			(ctx->constraints->min_height &&
584			ctx->constraints->min_height > frame->height))
585			return 0;
586
587			if (ctx->constraints->valid_sw_formats) {
588			enum AVPixelFormat *sw_formats = ctx->constraints->valid_sw_formats;
589			while (*sw_formats != AV_PIX_FMT_NONE) {
590			if (*sw_formats == src_hw_frame->sw_format)
591			break;
592			sw_formats++;
593			}
594			if (*sw_formats == AV_PIX_FMT_NONE)
595			return 0;
596			}
597
598			ctx->hw_frame_ref = av_hwframe_ctx_alloc(ctx->hw_device_ref);
599			if (!ctx->hw_frame_ref)
600			return AVERROR(ENOMEM);
601
602			hw_frame = (AVHWFramesContext *) ctx->hw_frame_ref->data;
603			hw_frame->format = AV_PIX_FMT_VULKAN;
604			hw_frame->sw_format = src_hw_frame->sw_format;
605			hw_frame->width = frame->width;
606			hw_frame->height = frame->height;
607
608			if (frame->format == AV_PIX_FMT_CUDA) {
609			vk_frame_ctx = hw_frame->hwctx;
610			vk_frame_ctx->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE;
611			}
612
613			ret = av_hwframe_ctx_init(ctx->hw_frame_ref);
614			if (ret < 0) {
615			av_log(renderer, AV_LOG_ERROR, "Create hwframe context failed, %s\n",
616			av_err2str(ret));
617			return ret;
618			}
619
620			av_hwframe_transfer_get_formats(ctx->hw_frame_ref,
621			AV_HWFRAME_TRANSFER_DIRECTION_TO,
622			&ctx->transfer_formats, 0);
623
624			return 0;
625			}
626
627			static inline int check_hw_transfer(RendererContext *ctx, AVFrame *frame)
628			{
629			if (!ctx->hw_frame_ref || !ctx->transfer_formats)
630			return 0;
631
632			for (int i = 0; ctx->transfer_formats[i] != AV_PIX_FMT_NONE; i++)
633			if (ctx->transfer_formats[i] == frame->format)
634			return 1;
635
636			return 0;
637			}
638
639			static inline int move_to_output_frame(RendererContext *ctx, AVFrame *frame)
640			{
641			int ret = av_frame_copy_props(ctx->vk_frame, frame);
642			if (ret < 0)
643			return ret;
644			av_frame_unref(frame);
645			av_frame_move_ref(frame, ctx->vk_frame);
646			return 0;
647			}
648
649			static int map_frame(VkRenderer *renderer, AVFrame *frame, int use_hw_frame)
650			{
651			RendererContext *ctx = (RendererContext *) renderer;
652			int ret;
653
654			if (use_hw_frame && !ctx->hw_frame_ref)
655			return AVERROR(ENOSYS);
656
657			// Try map data first
658			av_frame_unref(ctx->vk_frame);
659			if (use_hw_frame) {
660			ctx->vk_frame->hw_frames_ctx = av_buffer_ref(ctx->hw_frame_ref);
661			ctx->vk_frame->format = AV_PIX_FMT_VULKAN;
662			}
663			ret = av_hwframe_map(ctx->vk_frame, frame, 0);
664			if (!ret)
665			return move_to_output_frame(ctx, frame);
666
667			if (ret != AVERROR(ENOSYS))
668			av_log(NULL, AV_LOG_FATAL, "Map frame failed: %s\n", av_err2str(ret));
669			return ret;
670			}
671
672			static int transfer_frame(VkRenderer *renderer, AVFrame *frame, int use_hw_frame)
673			{
674			RendererContext *ctx = (RendererContext *) renderer;
675			int ret;
676
677			if (use_hw_frame && !check_hw_transfer(ctx, frame))
678			return AVERROR(ENOSYS);
679
680			av_frame_unref(ctx->vk_frame);
681			if (use_hw_frame)
682			av_hwframe_get_buffer(ctx->hw_frame_ref, ctx->vk_frame, 0);
683			ret = av_hwframe_transfer_data(ctx->vk_frame, frame, 1);
684			if (!ret)
685			return move_to_output_frame(ctx, frame);
686
687			if (ret != AVERROR(ENOSYS))
688			av_log(NULL, AV_LOG_FATAL, "Transfer frame failed: %s\n",
689			av_err2str(ret));
690			return ret;
691			}
692
693			static int convert_frame(VkRenderer *renderer, AVFrame *frame)
694			{
695			int ret;
696
697			if (!frame->hw_frames_ctx)
698			return 0;
699
700			if (frame->format == AV_PIX_FMT_VULKAN)
701			return 0;
702
703			ret = create_hw_frame(renderer, frame);
704			if (ret < 0)
705			return ret;
706
707			for (int use_hw = 1; use_hw >=0; use_hw--) {
708			ret = map_frame(renderer, frame, use_hw);
709			if (!ret)
710			return 0;
711			if (ret != AVERROR(ENOSYS))
712			return ret;
713
714			ret = transfer_frame(renderer, frame, use_hw);
715			if (!ret)
716			return 0;
717			if (ret != AVERROR(ENOSYS))
718			return ret;
719			}
720
721			return ret;
722			}
723
724			static int display(VkRenderer *renderer, AVFrame *frame, RenderParams *params)
725			{
726			SDL_Rect *rect = &params->target_rect;
727			struct pl_swapchain_frame swap_frame = {0};
728			struct pl_frame pl_frame = {0};
729			struct pl_frame target = {0};
730			struct pl_render_params pl_params = pl_render_default_params;
731			RendererContext *ctx = (RendererContext *) renderer;
732			int ret = 0;
733			struct pl_color_space hint = {0};
734
735			ret = convert_frame(renderer, frame);
736			if (ret < 0)
737			return ret;
738
739			if (!pl_map_avframe_ex(ctx->placebo_vulkan->gpu, &pl_frame, pl_avframe_params(
740			.frame = frame,
741			.tex = ctx->tex))) {
742			av_log(NULL, AV_LOG_ERROR, "pl_map_avframe_ex failed\n");
743			return AVERROR_EXTERNAL;
744			}
745
746			pl_color_space_from_avframe(&hint, frame);
747			pl_swapchain_colorspace_hint(ctx->swapchain, &hint);
748			if (!pl_swapchain_start_frame(ctx->swapchain, &swap_frame)) {
749			av_log(NULL, AV_LOG_ERROR, "start frame failed\n");
750			ret = AVERROR_EXTERNAL;
751			goto out;
752			}
753
754			pl_frame_from_swapchain(&target, &swap_frame);
755
756			target.crop = (pl_rect2df){.x0 = rect->x, .x1 = rect->x + rect->w,
757			.y0 = rect->y, .y1 = rect->y + rect->h};
758			switch (params->video_background_type) {
759			case VIDEO_BACKGROUND_TILES:
760			pl_params.background = PL_CLEAR_TILES;
761			pl_params.tile_size = VIDEO_BACKGROUND_TILE_SIZE * 2;
762			break;
763			case VIDEO_BACKGROUND_COLOR:
764			pl_params.background = PL_CLEAR_COLOR;
765			for (int i = 0; i < 3; i++)
766			pl_params.background_color[i] = params->video_background_color[i] / 255.0;
767			pl_params.background_transparency = (255 - params->video_background_color[3]) / 255.0;
768			break;
769			case VIDEO_BACKGROUND_NONE:
770			pl_frame.repr.alpha = PL_ALPHA_NONE;
771			break;
772			}
773
774			if (!pl_render_image(ctx->renderer, &pl_frame, &target, &pl_params)) {
775			av_log(NULL, AV_LOG_ERROR, "pl_render_image failed\n");
776			ret = AVERROR_EXTERNAL;
777			goto out;
778			}
779
780			if (!pl_swapchain_submit_frame(ctx->swapchain)) {
781			av_log(NULL, AV_LOG_ERROR, "pl_swapchain_submit_frame failed\n");
782			ret = AVERROR_EXTERNAL;
783			goto out;
784			}
785			pl_swapchain_swap_buffers(ctx->swapchain);
786
787			out:
788			pl_unmap_avframe(ctx->placebo_vulkan->gpu, &pl_frame);
789			return ret;
790			}
791
792			static int resize(VkRenderer *renderer, int width, int height)
793			{
794			RendererContext *ctx = (RendererContext *) renderer;
795
796			if (!pl_swapchain_resize(ctx->swapchain, &width, &height))
797			return AVERROR_EXTERNAL;
798			return 0;
799			}
800
801			static void destroy(VkRenderer *renderer)
802			{
803			RendererContext *ctx = (RendererContext *) renderer;
804			PFN_vkDestroySurfaceKHR vkDestroySurfaceKHR;
805
806			av_frame_free(&ctx->vk_frame);
807			av_freep(&ctx->transfer_formats);
808			av_hwframe_constraints_free(&ctx->constraints);
809			av_buffer_unref(&ctx->hw_frame_ref);
810
811			if (ctx->placebo_vulkan) {
812			for (int i = 0; i < FF_ARRAY_ELEMS(ctx->tex); i++)
813			pl_tex_destroy(ctx->placebo_vulkan->gpu, &ctx->tex[i]);
814			pl_renderer_destroy(&ctx->renderer);
815			pl_swapchain_destroy(&ctx->swapchain);
816			pl_vulkan_destroy(&ctx->placebo_vulkan);
817			}
818
819			if (ctx->vk_surface) {
820			vkDestroySurfaceKHR = (PFN_vkDestroySurfaceKHR)
821			ctx->get_proc_addr(ctx->inst, "vkDestroySurfaceKHR");
822			vkDestroySurfaceKHR(ctx->inst, ctx->vk_surface, NULL);
823			ctx->vk_surface = VK_NULL_HANDLE;
824			}
825
826			av_buffer_unref(&ctx->hw_device_ref);
827			pl_vk_inst_destroy(&ctx->placebo_instance);
828
829			pl_log_destroy(&ctx->vk_log);
830			}
831
832			static const AVClass vulkan_renderer_class = {
833			.class_name = "Vulkan Renderer",
834			.item_name = av_default_item_name,
835			.version = LIBAVUTIL_VERSION_INT,
836			};
837
838			VkRenderer *vk_get_renderer(void)
839			{
840			RendererContext *ctx = av_mallocz(sizeof(*ctx));
841			VkRenderer *renderer;
842
843			if (!ctx)
844			return NULL;
845
846			renderer = &ctx->api;
847			renderer->class = &vulkan_renderer_class;
848			renderer->get_hw_dev = get_hw_dev;
849			renderer->create = create;
850			renderer->display = display;
851			renderer->resize = resize;
852			renderer->destroy = destroy;
853
854			return renderer;
855			}
856
857			#else
858
859		✗	VkRenderer *vk_get_renderer(void)
860			{
861		✗	return NULL;
862			}
863
864			#endif
865
866		✗	int vk_renderer_create(VkRenderer *renderer, SDL_Window *window,
867			AVDictionary *opt)
868			{
869		✗	return renderer->create(renderer, window, opt);
870			}
871
872		✗	int vk_renderer_get_hw_dev(VkRenderer *renderer, AVBufferRef **dev)
873			{
874		✗	return renderer->get_hw_dev(renderer, dev);
875			}
876
877		✗	int vk_renderer_display(VkRenderer *renderer, AVFrame *frame, RenderParams *render_params)
878			{
879		✗	return renderer->display(renderer, frame, render_params);
880			}
881
882		✗	int vk_renderer_resize(VkRenderer *renderer, int width, int height)
883			{
884		✗	return renderer->resize(renderer, width, height);
885			}
886
887		✗	void vk_renderer_destroy(VkRenderer *renderer)
888			{
889		✗	renderer->destroy(renderer);
890		✗	}
891