FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Shared file cache protocol.
3			* Copyright (c) 2026 Niklas Haas
4			*
5			* This file is part of FFmpeg.
6			*
7			* FFmpeg is free software; you can redistribute it and/or
8			* modify it under the terms of the GNU Lesser General Public
9			* License as published by the Free Software Foundation; either
10			* version 2.1 of the License, or (at your option) any later version.
11			*
12			* FFmpeg is distributed in the hope that it will be useful,
13			* but WITHOUT ANY WARRANTY; without even the implied warranty of
14			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15			* Lesser General Public License for more details.
16			*
17			* You should have received a copy of the GNU Lesser General Public
18			* License along with FFmpeg; if not, write to the Free Software
19			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20			*
21			* Based on cache.c by Michael Niedermayer
22			*/
23
24			#include "libavutil/attributes.h"
25			#include "libavutil/avassert.h"
26			#include "libavutil/avstring.h"
27			#include "libavutil/crc.h"
28			#include "libavutil/error.h"
29			#include "libavutil/hash.h"
30			#include "libavutil/file_open.h"
31			#include "libavutil/mem.h"
32			#include "libavutil/opt.h"
33			#include "libavutil/time.h"
34
35			#include "internal.h"
36			#include "url.h"
37
38			#include <assert.h>
39			#include <errno.h>
40			#include <fcntl.h>
41			#include <inttypes.h>
42			#include <stdatomic.h>
43			#include <string.h>
44			#include <sys/file.h>
45			#include <sys/mman.h>
46			#include <sys/stat.h>
47			#include <unistd.h>
48
49			/**
50			* This hash should be resistant against collision attacks, so that an
51			* attacker could not generate e.g. two different URIs that map to the same
52			* cache file. This requires at least 64 bits of collision resistance in
53			* practice (i.e. 128 bits = 16 bytes of hash size). However, we can be
54			* conservative by computing e.g. a 256 bit hash and storing it inside the
55			* file header for verification.
56			*
57			* Note that due to the way we use atomics, we should avoid zero bytes in
58			* the resulting hash; hence we tweak the input slightly to avoid this.
59			* The resulting loss in hash strength is negligible, since 32 bytes is
60			* already much more than needed.
61			*/
62			#define HASH_METHOD "SHA512/256"
63			#define HASH_SIZE 32
64			#define HEADER_MAGIC MKTAG(u'\xFF', 'S', 'h', '$')
65			#define HEADER_VERSION 3
66
67			/**
68			* Hard watershed of consecutive failed blocks before we give up on the cache
69			* file altogether and assume it's entirely lost to us.
70			**/
71			#define MAX_CORRUPT_BLOCKS 10
72
73		✗	static int hash_uri(uint8_t hash[HASH_SIZE], const char *uri)
74			{
75		✗	struct AVHashContext *ctx = NULL;
76		✗	int ret = av_hash_alloc(&ctx, HASH_METHOD);
77		✗	if (ret < 0)
78		✗	return ret;
79
80		✗	const int16_t version = HEADER_VERSION;
81		✗	av_assert0(av_hash_get_size(ctx) == HASH_SIZE);
82		✗	av_hash_init(ctx);
83		✗	av_hash_update(ctx, (const uint8_t *) &version, sizeof(version));
84		✗	av_hash_update(ctx, (const uint8_t *) uri, strlen(uri));
85		✗	av_hash_final(ctx, hash);
86		✗	av_hash_freep(&ctx);
87
88		✗	for (int i = 0; i < HASH_SIZE; i++)
89		✗	hash[i] = hash[i] ? hash[i] : ~hash[i]; /* prevent zero bytes */
90		✗	return 0;
91			}
92
93			enum BlockState {
94			/* Reserved block state values */
95			BLOCK_NONE = 0, ///< block is not cached
96			BLOCK_PENDING, ///< a thread is currently trying to write this block
97			BLOCK_FAILED, ///< the underlying I/O source failed to read this block
98
99			/**
100			* All other block states represent valid cached blocks, with the value
101			* being the CRC of the block data.
102			*/
103			};
104
105		✗	static uint32_t get_block_crc(const uint8_t *block, size_t block_size)
106			{
107		✗	uint32_t crc = av_crc(av_crc_get_table(AV_CRC_32_IEEE), 0, block, block_size);
108		✗	switch (crc) {
109		✗	case BLOCK_NONE:
110			case BLOCK_FAILED:
111			case BLOCK_PENDING:
112		✗	return ~crc; /* avoid reserved block states */
113		✗	default:
114		✗	return crc;
115			}
116			}
117
118			typedef struct Block {
119			atomic_uint state; /* enum BlockState */
120			} Block;
121
122			typedef struct Spacemap {
123			atomic_uint header_magic;
124			atomic_ushort version;
125			atomic_ushort block_shift;
126			atomic_ullong filesize; /* byte offset of true EOF, or 0 if unknown */
127			atomic_uchar hash[HASH_SIZE]; /* hash of resource URI / filename */
128			atomic_ullong blocks_cached; /* (lower bound on) the number of blocks cached */
129			char reserved[72];
130
131			Block blocks[];
132			} Spacemap;
133
134			static_assert(offsetof(Spacemap, blocks) == 128, "Spacemap header layout mismatch");
135
136			/* Set to value iff the current value is unset (zero) */
137			#define DEF_SET_ONCE(ctype, atype) \
138			static int set_once_##atype(atomic_##atype *const ptr, const ctype value) \
139			{ \
140			ctype prev = 0; \
141			av_assert1(value != 0); \
142			if (atomic_compare_exchange_strong_explicit( \
143			ptr, &prev, value, memory_order_release, memory_order_relaxed)) \
144			return 1; \
145			else if (prev == value) \
146			return 0; \
147			else \
148			return AVERROR(EINVAL); \
149			}
150
151		✗	DEF_SET_ONCE(unsigned char, uchar)
152		✗	DEF_SET_ONCE(unsigned int, uint)
153		✗	DEF_SET_ONCE(unsigned short, ushort)
154		✗	DEF_SET_ONCE(unsigned long long, ullong)
155
156			typedef struct SharedContext {
157			AVClass *class;
158			URLContext *inner;
159			int64_t inner_pos;
160
161			/* options */
162			char *cache_dir;
163			int block_shift; ///< requested shift; updated on init if it disagrees
164			int read_only;
165			int64_t timeout;
166			int ignore_errors;
167			int retry_errors;
168			int retry_corrupt;
169			int verify;
170			int64_t cache_size_max;
171
172			/* misc state */
173			int64_t pos; ///< current logical position
174			uint8_t *tmp_buf;
175			int block_size;
176			int write_err; ///< write error occurred
177			int num_corrupt;
178			int64_t filesize; ///< once known
179			int64_t blocks_max; ///< maximum number of blocks to cache
180
181			/* cache file */
182			uint8_t *cache_data; ///< optional mmap of the cache file
183			char *cache_path;
184			off_t cache_size; ///< size of mapped memory region (for munmap)
185			int fd;
186
187			/* space map */
188			Spacemap *spacemap;
189			char *map_path;
190			off_t map_size;
191			int mapfd;
192
193			/* statistics */
194			int64_t nb_hit;
195			int64_t nb_miss;
196			} SharedContext;
197
198		✗	static int shared_close(URLContext *h)
199			{
200		✗	SharedContext *s = h->priv_data;
201
202		✗	ffurl_close(s->inner);
203		✗	if (s->cache_data)
204		✗	munmap(s->cache_data, s->cache_size);
205		✗	if (s->spacemap)
206		✗	munmap(s->spacemap, s->map_size);
207		✗	if (s->fd != -1)
208		✗	close(s->fd);
209		✗	if (s->mapfd != -1)
210		✗	close(s->mapfd);
211		✗	av_freep(&s->cache_path);
212		✗	av_freep(&s->map_path);
213		✗	av_freep(&s->tmp_buf);
214
215		✗	av_log(h, AV_LOG_DEBUG, "Cache statistics: %"PRId64" hits, %"PRId64" misses\n",
216			s->nb_hit, s->nb_miss);
217		✗	return 0;
218			}
219
220			static int cache_map(URLContext *h, int64_t filesize);
221			static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE]);
222			static int spacemap_grow(URLContext *h, int64_t block);
223
224		✗	static int64_t get_filesize(URLContext *h)
225			{
226		✗	SharedContext *s = h->priv_data;
227		✗	if (!s->filesize) {
228		✗	uint64_t size = atomic_load_explicit(&s->spacemap->filesize, memory_order_relaxed);
229		✗	if (size > INT64_MAX)
230		✗	return AVERROR(EINVAL);
231		✗	else if (size)
232		✗	s->filesize = size;
233			}
234
235		✗	return s->filesize;
236			}
237
238		✗	static int set_filesize(URLContext *h, int64_t new_size)
239			{
240		✗	SharedContext *s = h->priv_data;
241			int ret;
242
243		✗	if (!new_size)
244		✗	return 0;
245
246		✗	ret = set_once_ullong(&s->spacemap->filesize, new_size);
247		✗	if (ret < 0) {
248		✗	av_log(h, AV_LOG_ERROR, "Cached file size mismatch, expected: "
249			"%"PRId64", got: %"PRIu64"!\n", new_size,
250		✗	(uint64_t) atomic_load(&s->spacemap->filesize));
251		✗	return ret;
252		✗	} else if (ret) {
253			/* Opportunistically map the file; this also sets the correct filesize.
254			* Ignore errors as this is not critical to the cache logic. */
255		✗	cache_map(h, new_size);
256			}
257
258		✗	return ret;
259			}
260
261		✗	static int is_ignorable_error(int64_t err)
262			{
263		✗	switch (err) {
264		✗	case AVERROR_EXIT:
265			case AVERROR_EOF:
266			case AVERROR_BUG:
267			case AVERROR(EAGAIN):
268			case AVERROR(ENOSYS):
269			case AVERROR(EINVAL):
270		✗	return 0;
271		✗	default:
272		✗	return err < 0;
273			}
274			}
275
276		✗	static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary **options)
277			{
278		✗	SharedContext *s = h->priv_data;
279			int ret;
280
281		✗	if (!s->cache_dir || !s->cache_dir[0]) {
282		✗	av_log(h, AV_LOG_ERROR, "Missing path for shared cache! Specify a "
283			"directory using the -cache_dir option.\n");
284		✗	return AVERROR(EINVAL);
285			}
286
287		✗	s->fd = s->mapfd = -1; /* Set these early for shared_close() failure path */
288
289			/* Open underlying protocol */
290		✗	av_strstart(arg, "shared:", &arg);
291		✗	ret = ffurl_open_whitelist(&s->inner, arg, flags, &h->interrupt_callback,
292			options, h->protocol_whitelist, h->protocol_blacklist, h);
293		✗	if (is_ignorable_error(ret) && s->ignore_errors) {
294		✗	av_log(h, AV_LOG_WARNING, "Underlying URL failed to open: %s. "
295		✗	"Continuing with cache file only.\n", av_err2str(ret));
296		✗	} else if (ret < 0)
297		✗	goto fail;
298
299			uint8_t hash[HASH_SIZE];
300		✗	ret = hash_uri(hash, arg);
301		✗	if (ret < 0)
302		✗	goto fail;
303
304			/* 128 bits is enough for collision resistance; we already store the full
305			* hash inside the header for verification */
306			char filename[2 * 16 + 1];
307		✗	ff_data_to_hex(filename, hash, sizeof(filename)/2, 0);
308		✗	s->cache_path = av_asprintf("%s/%s.cache", s->cache_dir, filename);
309		✗	s->map_path = av_asprintf("%s/%s.spacemap", s->cache_dir, filename);
310		✗	if (!s->cache_path || !s->map_path) {
311		✗	ret = AVERROR(ENOMEM);
312		✗	goto fail;
313			}
314
315		✗	av_log(h, AV_LOG_VERBOSE, "Opening cache file '%s' for URI: '%s'\n",
316		✗	s->cache_path, s->inner ? s->inner->filename : arg);
317
318		✗	const int mode = O_RDWR | (s->inner ? O_CREAT : 0);
319		✗	s->fd = avpriv_open(s->cache_path, mode, 0660);
320		✗	s->mapfd = s->fd >= 0 ? avpriv_open(s->map_path, mode, 0660) : -1;
321		✗	if (s->fd < 0 || s->mapfd < 0) {
322		✗	ret = AVERROR(errno);
323		✗	av_log(h, AV_LOG_ERROR, "Failed to open '%s': %s\n",
324		✗	s->fd < 0 ? s->cache_path : s->map_path, av_err2str(ret));
325		✗	goto fail;
326			}
327
328		✗	ret = spacemap_init(h, hash);
329		✗	if (ret < 0)
330		✗	goto fail;
331
332			/* s->block_shift is fully settled after spacemap_init() */
333		✗	s->block_size = 1 << s->block_shift;
334		✗	s->blocks_max = s->cache_size_max >> s->block_shift;
335
336		✗	int64_t filesize = get_filesize(h);
337		✗	if (filesize < 0) {
338		✗	ret = (int) filesize;
339		✗	goto fail;
340		✗	} else if (!filesize) {
341			/* Filesize is not yet known, try to get it from the underlying URL;
342			* go through our own seek function to handle errors and updates */
343		✗	filesize = ffurl_size(h);
344		✗	if (filesize < 0 && filesize != AVERROR(ENOSYS)) {
345		✗	ret = (int) filesize;
346		✗	goto fail;
347			}
348			}
349
350		✗	if (filesize > 0) {
351		✗	int64_t last_pos = filesize - 1;
352		✗	int64_t last_block = last_pos >> s->block_shift;
353		✗	ret = spacemap_grow(h, last_block);
354		✗	if (ret < 0)
355		✗	goto fail;
356
357			/* If filesize is known, we can directly mmap() the cache file */
358		✗	ret = cache_map(h, filesize);
359		✗	if (ret < 0) {
360		✗	av_log(h, AV_LOG_WARNING, "Failed to map cache file: %s. Falling "
361		✗	"back to normal read/write\n", av_err2str(ret));
362		✗	ret = 0;
363			}
364			}
365
366			/* Temporary buffer needed for pread/pwrite() fallback */
367		✗	s->tmp_buf = av_malloc(s->block_size);
368		✗	if (!s->tmp_buf) {
369		✗	ret = AVERROR(ENOMEM);
370		✗	goto fail;
371			}
372
373		✗	h->max_packet_size = s->block_size;
374		✗	h->min_packet_size = s->block_size;
375		✗	ret = 0;
376
377		✗	fail:
378		✗	if (ret < 0)
379		✗	shared_close(h);
380		✗	return ret;
381			}
382
383		✗	static int cache_map(URLContext *h, int64_t filesize)
384			{
385		✗	SharedContext *s = h->priv_data;
386		✗	if (s->cache_size >= filesize || filesize > SIZE_MAX)
387		✗	return 0;
388
389		✗	if (s->cache_data) {
390		✗	munmap(s->cache_data, s->cache_size);
391		✗	s->cache_data = NULL;
392		✗	s->cache_size = 0;
393			}
394
395			struct stat st;
396		✗	int ret = fstat(s->fd, &st);
397		✗	if (ret < 0)
398		✗	return AVERROR(errno);
399
400		✗	if (st.st_size != filesize) {
401			/* Ensure the file size is correct before mapping; this can happen if
402			* another process wrote the correct filesize to the header but
403			* crashed right before actually successfully resizing the file. */
404		✗	ret = ftruncate(s->fd, filesize);
405		✗	if (ret < 0)
406		✗	return AVERROR(errno);
407			}
408
409		✗	s->cache_data = mmap(NULL, filesize, PROT_READ | PROT_WRITE, MAP_SHARED, s->fd, 0);
410		✗	if (s->cache_data == MAP_FAILED) {
411		✗	s->cache_data = NULL;
412		✗	return AVERROR(errno);
413			}
414
415		✗	s->cache_size = filesize;
416		✗	return 0;
417			}
418
419		✗	static int spacemap_remap(URLContext *h, size_t map_size)
420			{
421		✗	SharedContext *s = h->priv_data;
422		✗	int ret, did_grow = 0, locked = 0;
423		✗	if (map_size <= s->map_size)
424		✗	return 0;
425
426			/* Opportunistically get current filesize before attempting to lock */
427			struct stat st;
428		✗	ret = fstat(s->mapfd, &st);
429		✗	if (ret < 0) {
430		✗	ret = AVERROR(errno);
431		✗	goto fail;
432			}
433
434		✗	if (st.st_size >= map_size)
435		✗	goto skip_resize;
436
437			/* Lock the spacemap to ensure nobody else is currently resizing it */
438		✗	ret = flock(s->mapfd, LOCK_EX);
439		✗	if (ret < 0) {
440		✗	ret = AVERROR(errno);
441		✗	goto fail;
442			}
443		✗	locked = 1;
444
445			/* Refresh filesize after acquiring the lock */
446		✗	ret = fstat(s->mapfd, &st);
447		✗	if (ret < 0) {
448		✗	ret = AVERROR(errno);
449		✗	goto fail;
450			}
451
452		✗	if (st.st_size >= map_size)
453		✗	goto skip_resize;
454
455		✗	ret = ftruncate(s->mapfd, map_size);
456		✗	if (ret < 0) {
457		✗	ret = AVERROR(errno);
458		✗	goto fail;
459			}
460		✗	st.st_size = map_size;
461		✗	did_grow = 1;
462
463		✗	skip_resize:
464		✗	if (s->spacemap)
465		✗	munmap(s->spacemap, s->map_size);
466		✗	s->map_size = st.st_size;
467		✗	s->spacemap = mmap(NULL, s->map_size, PROT_READ | PROT_WRITE, MAP_SHARED, s->mapfd, 0);
468		✗	if (s->spacemap == MAP_FAILED) {
469		✗	s->spacemap = NULL; /* for munmap check */
470		✗	s->map_size = 0;
471		✗	ret = AVERROR(errno);
472		✗	goto fail;
473			}
474
475		✗	if (locked) {
476		✗	flock(s->mapfd, LOCK_UN);
477		✗	locked = 0;
478			}
479
480		✗	return did_grow;
481
482		✗	fail:
483		✗	if (locked)
484		✗	flock(s->mapfd, LOCK_UN);
485		✗	av_log(h, AV_LOG_ERROR, "Failed to resize space map: %s\n", av_err2str(ret));
486		✗	return ret;
487			}
488
489		✗	static int spacemap_grow(URLContext *h, int64_t block)
490			{
491		✗	SharedContext *s = h->priv_data;
492		✗	int64_t num_blocks = block + 1;
493		✗	size_t map_bytes = sizeof(Spacemap) + num_blocks * sizeof(Block);
494
495			/* When streaming files without known size, round up the number of blocks
496			* to the nearest multiple of the block size to reduce the rate of resizes */
497		✗	int64_t filesize = get_filesize(h);
498		✗	if (filesize < 0)
499		✗	return (int) filesize;
500		✗	else if (!filesize) {
501		✗	av_assert0(s->block_size > 0);
502		✗	map_bytes = FFALIGN(map_bytes, (int64_t) s->block_size);
503			}
504
505		✗	if (map_bytes < num_blocks)
506		✗	return AVERROR(EINVAL); /* overflow */
507
508		✗	const off_t old_size = s->map_size;
509		✗	int ret = spacemap_remap(h, map_bytes);
510		✗	if (ret < 0)
511		✗	return ret;
512
513			/* Report new size after successful grow */
514		✗	if (s->map_size > old_size) {
515		✗	num_blocks = (s->map_size - sizeof(Spacemap)) / sizeof(Block);
516		✗	av_log(h, AV_LOG_DEBUG,
517			"%s %zu bytes, capacity: %"PRId64" blocks = %"PRId64" MB\n",
518			ret ? "Resized spacemap to" : "Mapped spacemap with",
519		✗	(size_t) s->map_size, num_blocks,
520		✗	(num_blocks * (int64_t) s->block_size) >> 20);
521			}
522		✗	return 0;
523			}
524
525		✗	static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE])
526			{
527		✗	SharedContext *s = h->priv_data;
528			int ret;
529
530		✗	ret = spacemap_remap(h, sizeof(Spacemap));
531		✗	if (ret < 0)
532		✗	return ret;
533
534		✗	if ((ret = set_once_uint(&s->spacemap->header_magic, HEADER_MAGIC)) < 0 ||
535		✗	(ret = set_once_ushort(&s->spacemap->version, HEADER_VERSION)) < 0)
536			{
537		✗	av_log(h, AV_LOG_ERROR, "Shared cache spacemap header mismatch!\n");
538		✗	av_log(h, AV_LOG_ERROR, " Expected magic: 0x%X, version: %d\n",
539			HEADER_MAGIC, HEADER_VERSION);
540		✗	av_log(h, AV_LOG_ERROR, " Got magic: 0x%X, version: %d\n",
541		✗	atomic_load(&s->spacemap->header_magic),
542		✗	atomic_load(&s->spacemap->version));
543		✗	return ret;
544			}
545
546		✗	ret = set_once_ushort(&s->spacemap->block_shift, s->block_shift);
547		✗	if (ret < 0) {
548		✗	const int shift = atomic_load(&s->spacemap->block_shift);
549		✗	av_log(h, AV_LOG_WARNING, "Shared cache uses block shift %d, "
550			"but requested block shift is %d.\n", shift, s->block_shift);
551		✗	if (shift < 9 || shift > 30) {
552		✗	av_log(h, AV_LOG_ERROR, "Invalid block shift %d in cache file!\n", shift);
553		✗	return AVERROR(EINVAL);
554			}
555		✗	s->block_shift = shift;
556			}
557
558		✗	for (int i = 0; i < HASH_SIZE; i++) {
559		✗	ret = set_once_uchar(&s->spacemap->hash[i], hash[i]);
560		✗	if (ret < 0) {
561		✗	av_log(h, AV_LOG_ERROR, "Shared cache spacemap hash mismatch!\n");
562			char hash_hex[2 * HASH_SIZE + 1];
563		✗	ff_data_to_hex(hash_hex, hash, HASH_SIZE, 0);
564		✗	av_log(h, AV_LOG_ERROR, " Expected hash: %s\n", hash_hex);
565			uint8_t hash2[HASH_SIZE];
566		✗	for (int j = 0; j < HASH_SIZE; ++j)
567		✗	hash2[j] = atomic_load_explicit(&s->spacemap->hash[j], memory_order_relaxed);
568		✗	ff_data_to_hex(hash_hex, hash2, HASH_SIZE, 0);
569		✗	av_log(h, AV_LOG_ERROR, " Got hash: %s\n", hash_hex);
570		✗	return ret;
571			}
572			}
573
574		✗	if (ret) /* set_once() return 1 if this is the first time setting the value */
575		✗	av_log(h, AV_LOG_DEBUG, "Initialized new cache spacemap.\n");
576
577		✗	return ret;
578			}
579
580		✗	static int read_cache(SharedContext *s, uint8_t *buf, size_t size, off_t offset)
581			{
582		✗	if (s->cache_data) {
583			av_assert1(offset + size <= s->cache_size);
584		✗	memcpy(buf, s->cache_data + offset, size);
585		✗	return 0;
586			}
587
588		✗	while (size) {
589		✗	ssize_t ret = pread(s->fd, buf, size, offset);
590		✗	if (ret <= 0)
591		✗	return ret ? AVERROR(errno) : AVERROR_EOF;
592		✗	buf += ret;
593		✗	offset += ret;
594		✗	size -= ret;
595			}
596
597		✗	return 0;
598			}
599
600		✗	static int write_cache(SharedContext *s, const uint8_t *buf, size_t size, off_t offset)
601			{
602		✗	if (s->cache_data) {
603			av_assert1(offset + size <= s->cache_size);
604		✗	memcpy(s->cache_data + offset, buf, size);
605		✗	return 0;
606			}
607
608		✗	while (size) {
609		✗	ssize_t ret = pwrite(s->fd, buf, size, offset);
610		✗	if (ret <= 0)
611		✗	return ret ? AVERROR(errno) : AVERROR(EIO);
612		✗	buf += ret;
613		✗	offset += ret;
614		✗	size -= ret;
615			}
616
617		✗	return 0;
618			}
619
620		✗	static int clamp_size(URLContext *h, int size, int64_t pos, int64_t filesize)
621			{
622		✗	if (!filesize)
623		✗	return size;
624		✗	else if (pos > filesize)
625		✗	return 0;
626			else
627		✗	return FFMIN(filesize - pos, size);
628			}
629
630		✗	static int shared_read(URLContext *h, unsigned char *buf, int size)
631			{
632		✗	SharedContext *s = h->priv_data;
633			uint8_t *tmp;
634			int ret;
635		✗	if (!s->spacemap)
636		✗	return AVERROR(EIO);
637
638		✗	if (size <= 0)
639		✗	return 0;
640
641		✗	int64_t filesize = get_filesize(h);
642		✗	if (filesize < 0)
643		✗	return (int) filesize;
644
645		✗	size = clamp_size(h, size, s->pos, filesize);
646		✗	if (size <= 0)
647		✗	return AVERROR_EOF;
648
649		✗	const int64_t block_id = s->pos >> s->block_shift;
650		✗	const int64_t offset = s->pos & (s->block_size - 1);
651		✗	const int64_t block_pos = block_id * s->block_size;
652		✗	int block_size = clamp_size(h, s->block_size, block_pos, filesize);
653		✗	ret = spacemap_grow(h, block_id);
654		✗	if (ret < 0)
655		✗	return ret;
656
657		✗	Block *const block = &s->spacemap->blocks[block_id];
658		✗	unsigned state = atomic_load_explicit(&block->state, memory_order_acquire);
659		✗	int64_t pending_since = 0;
660		✗	int verify_read = 0, acquired = 0, allocated = 0;
661
662		✗	retry:
663		✗	switch (state) {
664		✗	default:
665		✗	if (s->num_corrupt >= MAX_CORRUPT_BLOCKS)
666		✗	goto read_block; /* assume broken cache file */
667
668			/* filesize may have become known in the meantime */
669		✗	filesize = get_filesize(h);
670		✗	if (filesize < 0)
671		✗	return (int) filesize;
672
673			/* We always need to read the entire block to verify integrity */
674		✗	block_size = clamp_size(h, block_size, block_pos, filesize);
675		✗	if (s->cache_data) {
676			av_assert1(block_pos + block_size <= s->cache_size);
677		✗	tmp = s->cache_data + block_pos;
678			} else {
679		✗	tmp = s->tmp_buf;
680		✗	ret = read_cache(s, tmp, block_size, block_pos);
681		✗	if (ret < 0) {
682		✗	av_log(h, AV_LOG_ERROR, "Failed to read from cache file: %s\n", av_err2str(ret));
683		✗	if (ret == AVERROR_EOF) { /* e.g. cache appears truncated? */
684		✗	if (s->retry_corrupt) {
685		✗	s->num_corrupt++;
686		✗	goto read_block;
687			}
688		✗	ret = AVERROR(EIO); /* don't propagate EOF to caller */
689			}
690		✗	return ret;
691			}
692			}
693
694		✗	uint32_t crc = get_block_crc(tmp, block_size);
695		✗	if (crc != state) {
696		✗	av_log(h, AV_LOG_ERROR, "Cache corruption detected for block 0x%"PRIx64" at "
697			"offset 0x%"PRIx64": expected CRC: 0x%08X, got: 0x%08X\n",
698			block_id, block_pos, state, crc);
699		✗	if (s->retry_corrupt) {
700		✗	s->num_corrupt++;
701		✗	goto read_block;
702			}
703		✗	return AVERROR(EIO);
704			} else
705		✗	s->num_corrupt = 0; /* reset corrupt block count on success */
706
707		✗	tmp += (ptrdiff_t) offset;
708		✗	size = FFMIN(size, block_size - offset);
709		✗	if (size <= 0)
710		✗	return AVERROR_EOF;
711		✗	if (s->verify) {
712		✗	verify_read = 1;
713		✗	break; /* fall through to the cache miss logic */
714			}
715
716		✗	memcpy(buf, tmp, size);
717		✗	s->nb_hit++;
718		✗	s->pos += size;
719		✗	return size;
720
721		✗	case BLOCK_FAILED:
722		✗	if (s->retry_errors)
723		✗	goto read_block;
724		✗	return AVERROR(EIO);
725
726		✗	read_block:
727		✗	if (s->num_corrupt == MAX_CORRUPT_BLOCKS) {
728		✗	av_log(h, AV_LOG_ERROR, "Too many consecutive corrupt blocks; "
729			"assuming cache file is completely broken.\n");
730		✗	s->num_corrupt++; /* silence this log on subsequent reads */
731			}
732			av_fallthrough;
733
734			case BLOCK_NONE:
735		✗	if (s->read_only || s->write_err || !s->inner)
736			break; /* don't mark block as pending */
737		✗	else if (s->cache_size_max) {
738		✗	int64_t cached = atomic_load_explicit(&s->spacemap->blocks_cached,
739			memory_order_relaxed);
740		✗	if (cached >= s->blocks_max) {
741		✗	av_log(h, AV_LOG_WARNING, "Cache size limit reached (%"PRId64" "
742			"blocks = %"PRId64" bytes), switching to read-only mode.\n",
743		✗	s->blocks_max, s->blocks_max << s->block_shift);
744		✗	s->read_only = 1;
745		✗	break;
746			}
747			}
748
749		✗	if (atomic_compare_exchange_strong_explicit(&block->state, &state,
750			BLOCK_PENDING,
751			memory_order_acquire,
752			memory_order_acquire))
753			{
754			/* Acquired pending state, proceed to fetch the block */
755		✗	acquired = 1;
756		✗	allocated = (state == BLOCK_NONE || state == BLOCK_FAILED);
757		✗	state = BLOCK_PENDING;
758		✗	break;
759			}
760			/* CAS failed, another thread changed the state; reload it */
761		✗	goto retry;
762
763		✗	case BLOCK_PENDING:
764			/* Another thread is busy fetching this block, wait for it to finish */
765		✗	if (!s->timeout) {
766		✗	break; /* no timeout requested, immediately race to fetch block */
767		✗	} else if (pending_since) {
768		✗	int64_t new = av_gettime_relative();
769		✗	if (new - pending_since >= s->timeout)
770		✗	break; /* timeout expired, try to fetch the block ourselves */
771			} else {
772		✗	pending_since = av_gettime_relative();
773			}
774
775		✗	if (h->flags & AVIO_FLAG_NONBLOCK)
776		✗	return AVERROR(EAGAIN);
777
778			/* Make sure we try a few times before giving up */
779		✗	av_usleep(FFMIN(s->timeout >> 4, 10000));
780		✗	if (ff_check_interrupt(&h->interrupt_callback))
781		✗	return AVERROR_EXIT;
782
783		✗	state = atomic_load_explicit(&block->state, memory_order_acquire);
784		✗	goto retry;
785			}
786
787			/* Release pending state on failure to avoid stalling other threads */
788			#define RELEASE_PENDING(block, state) \
789			do { \
790			if (acquired) { \
791			av_assert1(state == BLOCK_PENDING); \
792			atomic_compare_exchange_strong_explicit( \
793			&block->state, &state, BLOCK_NONE, memory_order_relaxed, \
794			memory_order_relaxed); \
795			} \
796			} while (0)
797
798			/* Cache miss, fetch this block from underlying protocol */
799		✗	s->nb_miss++;
800
801		✗	if (!s->inner) {
802		✗	av_log(h, AV_LOG_ERROR, "Cache miss for block 0x%"PRIx64" at offset "
803			"0x%"PRIx64", but underlying protocol is not available!\n",
804			block_id, block_pos);
805		✗	av_assert0(!acquired);
806		✗	return AVERROR(EIO);
807			}
808
809		✗	const int read_only = s->read_only || s->write_err || verify_read;
810		✗	int64_t inner_pos = read_only ? s->pos : block_pos;
811		✗	if (s->inner_pos != inner_pos) {
812		✗	inner_pos = ffurl_seek(s->inner, inner_pos, SEEK_SET);
813		✗	if (inner_pos < 0) {
814		✗	av_log(h, AV_LOG_ERROR, "Failed to seek underlying protocol: %s\n",
815		✗	av_err2str(inner_pos));
816		✗	RELEASE_PENDING(block, state);
817		✗	return inner_pos;
818			}
819
820		✗	av_log(h, AV_LOG_DEBUG, "Inner seek to 0x%"PRIx64"\n", inner_pos);
821		✗	s->inner_pos = inner_pos;
822			}
823
824		✗	if (read_only) {
825			/* Directly defer to the underlying protocol */
826		✗	ret = ffurl_read(s->inner, buf, size);
827		✗	if (ret < 0) {
828			av_assert1(!acquired);
829		✗	return ret;
830			} else {
831		✗	s->inner_pos = inner_pos + ret;
832			}
833
834			/* Verify the read data against the cached data if requested */
835		✗	if (verify_read && memcmp(buf, tmp, ret)) {
836		✗	av_log(h, AV_LOG_ERROR, "Cache verification failed for %d bytes "
837			"in block 0x%"PRIx64" at offset 0x%"PRIx64" + %"PRId64"!\n",
838			ret, block_id, block_pos, offset);
839		✗	return AVERROR(EIO);
840			}
841
842		✗	s->pos = s->inner_pos;
843		✗	return ret;
844			}
845
846		✗	int write_back = 1;
847		✗	if (s->cache_data && acquired) {
848			/* Read directly into memory mapped cache file */
849		✗	tmp = s->cache_data + block_pos;
850		✗	write_back = 0;
851		✗	} else if (size >= block_size && !offset) {
852			/* Read directly into output buffer if aligned and large enough */
853		✗	tmp = buf;
854			} else {
855			/* Read into temporary buffer and copy later */
856		✗	tmp = s->tmp_buf;
857			}
858
859			/* Try and fetch the entire block */
860		✗	av_assert0(inner_pos == block_pos);
861		✗	int bytes_read = 0;
862		✗	while (bytes_read < block_size) {
863		✗	ret = ffurl_read(s->inner, &tmp[bytes_read], block_size - bytes_read);
864		✗	if (!ret || ret == AVERROR_EOF)
865			break;
866		✗	else if (ret < 0) {
867		✗	av_log(h, AV_LOG_ERROR, "Failed to read block 0x%"PRIx64": %s\n",
868		✗	block_id, av_err2str(ret));
869		✗	if (ret == AVERROR(EAGAIN) || ret == AVERROR_EXIT) {
870		✗	RELEASE_PENDING(block, state);
871		✗	return ret; /* transient error, allow retries */
872			}
873
874			/* Try to mark block as failed; ignore errors - any mismatch
875			* here will mean that either another thread already marked it
876			* as failed, or successfully cached it in the meantime */
877		✗	atomic_compare_exchange_strong_explicit(&block->state, &state,
878			BLOCK_FAILED,
879			memory_order_relaxed,
880			memory_order_relaxed);
881		✗	return ret;
882			}
883
884		✗	bytes_read += ret;
885		✗	s->inner_pos += ret;
886			}
887
888		✗	if (bytes_read < block_size) {
889			/* Learned location of true EOF, update filesize */
890		✗	ret = set_filesize(h, inner_pos + bytes_read);
891		✗	if (ret < 0) {
892		✗	RELEASE_PENDING(block, state);
893		✗	return ret;
894			}
895			}
896
897		✗	if (bytes_read > 0) {
898		✗	ret = write_back ? write_cache(s, tmp, bytes_read, block_pos) : 0;
899		✗	if (ret < 0) {
900		✗	if (ret != AVERROR(EINTR)) {
901		✗	av_log(h, AV_LOG_ERROR, "Failed to write to cache file: %s\n",
902		✗	av_err2str(ret));
903		✗	s->write_err = 1;
904			}
905		✗	RELEASE_PENDING(block, state);
906			} else {
907		✗	uint32_t crc = get_block_crc(tmp, bytes_read);
908		✗	av_log(h, AV_LOG_TRACE, "Cached %d bytes to block 0x%"PRIx64" at "
909			"offset 0x%"PRIx64", CRC 0x%08X\n", bytes_read, block_id,
910			block_pos, crc);
911		✗	atomic_store_explicit(&block->state, crc, memory_order_release);
912		✗	if (allocated)
913		✗	atomic_fetch_add_explicit(&s->spacemap->blocks_cached, 1, memory_order_release);
914			}
915			} else {
916		✗	RELEASE_PENDING(block, state);
917		✗	return AVERROR_EOF;
918			}
919
920		✗	size = FFMIN(bytes_read - offset, size);
921		✗	if (size <= 0)
922		✗	return AVERROR_EOF;
923		✗	if (tmp != buf)
924		✗	memcpy(buf, &tmp[offset], size);
925		✗	s->pos += size;
926		✗	return size;
927			}
928
929		✗	static int64_t shared_seek(URLContext *h, int64_t pos, int whence)
930			{
931		✗	SharedContext *s = h->priv_data;
932			int64_t res;
933		✗	if (!s->spacemap)
934		✗	return AVERROR(EIO);
935
936		✗	const int64_t filesize = get_filesize(h);
937		✗	if (filesize < 0)
938		✗	return filesize;
939
940		✗	switch (whence) {
941		✗	case AVSEEK_SIZE:
942		✗	if (filesize)
943		✗	return filesize;
944		✗	res = s->inner ? ffurl_seek(s->inner, pos, whence) : AVERROR(ENOSYS);
945		✗	if (res > 0) {
946		✗	if (set_filesize(h, res) < 0)
947		✗	return AVERROR(EINVAL);
948		✗	} else if (is_ignorable_error(res) && s->ignore_errors) {
949		✗	av_log(h, AV_LOG_WARNING, "Underlying URL failed to get size: %s. "
950		✗	"Continuing with cache file only.\n", av_err2str(res));
951		✗	ffurl_closep(&s->inner);
952		✗	res = AVERROR(ENOSYS);
953			}
954		✗	return res;
955		✗	case SEEK_SET:
956		✗	break;
957		✗	case SEEK_CUR:
958		✗	pos += s->pos;
959		✗	break;
960		✗	case SEEK_END:
961		✗	if (filesize) {
962		✗	pos += filesize;
963		✗	break;
964			}
965
966			/* Defer to underlying protocol if filesize is unknown */
967		✗	res = s->inner ? ffurl_seek(s->inner, pos, whence) : AVERROR(ENOSYS);
968		✗	if (is_ignorable_error(res) && s->ignore_errors) {
969		✗	av_log(h, AV_LOG_WARNING, "Underlying URL failed to seek: %s. "
970		✗	"Continuing with cache file only.\n", av_err2str(res));
971		✗	ffurl_closep(&s->inner);
972		✗	return AVERROR(ENOSYS);
973		✗	} else if (res < 0)
974		✗	return res;
975
976			/* Opportunistically update known filesize */
977		✗	if (set_filesize(h, res - pos) < 0)
978		✗	return AVERROR(EINVAL);
979		✗	av_log(h, AV_LOG_DEBUG, "Inner seek to 0x%"PRIx64"\n", res);
980		✗	return s->pos = s->inner_pos = res;
981		✗	default:
982		✗	return AVERROR(EINVAL);
983			}
984
985		✗	if (pos < 0)
986		✗	return AVERROR(EINVAL);
987
988		✗	av_log(h, AV_LOG_DEBUG, "Virtual seek to 0x%"PRIx64"\n", pos);
989		✗	return s->pos = pos;
990			}
991
992		✗	static int shared_get_file_handle(URLContext *h)
993			{
994		✗	SharedContext *s = h->priv_data;
995		✗	return s->inner ? ffurl_get_file_handle(s->inner) : -1;
996			}
997
998		✗	static int shared_get_short_seek(URLContext *h)
999			{
1000		✗	SharedContext *s = h->priv_data;
1001		✗	int ret = s->inner ? ffurl_get_short_seek(s->inner) : 0;
1002		✗	return ret > 0 ? FFMAX(ret, s->block_size) : s->block_size;
1003			}
1004
1005			#define OFFSET(x) offsetof(SharedContext, x)
1006			#define D AV_OPT_FLAG_DECODING_PARAM
1007
1008			static const AVOption options[] = {
1009			{ "cache_dir", "Directory path for shared file cache", OFFSET(cache_dir), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D },
1010			{ "block_shift", "Set the base 2 logarithm of the block size", OFFSET(block_shift), AV_OPT_TYPE_INT, {.i64 = 15}, 9, 30, .flags = D },
1011			{ "read_only", "Don't write data to the cache, only read from it", OFFSET(read_only), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
1012			{ "cache_verify", "Verify correctness of the cache against the source", OFFSET(verify), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
1013			{ "cache_timeout", "Time in us to wait before re-fetching pending blocks", OFFSET(timeout), AV_OPT_TYPE_INT64, {.i64 = 10000}, 0, INT64_MAX, .flags = D },
1014			{ "ignore_errors", "Continue even if the inner URL failed", OFFSET(ignore_errors), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
1015			{ "retry_errors", "Re-request blocks even if they previously failed", OFFSET(retry_errors), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
1016			{ "retry_corrupt", "Re-request blocks that fail the CRC check", OFFSET(retry_corrupt), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
1017			{ "cache_size_max", "Limit the maximum amount of data cached", OFFSET(cache_size_max), AV_OPT_TYPE_INT64, {.i64 = 0}, 0, INT64_MAX, .flags = D },
1018			{0},
1019			};
1020
1021			static const AVClass shared_context_class = {
1022			.class_name = "shared",
1023			.item_name = av_default_item_name,
1024			.option = options,
1025			.version = LIBAVUTIL_VERSION_INT,
1026			};
1027
1028			const URLProtocol ff_shared_protocol = {
1029			.name = "shared",
1030			.url_open2 = shared_open,
1031			.url_read = shared_read,
1032			.url_seek = shared_seek,
1033			.url_close = shared_close,
1034			.url_get_file_handle = shared_get_file_handle,
1035			.url_get_short_seek = shared_get_short_seek,
1036			.priv_data_size = sizeof(SharedContext),
1037			.priv_data_class = &shared_context_class,
1038			};
1039