FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Shared file cache protocol.
3			* Copyright (c) 2026 Niklas Haas
4			*
5			* This file is part of FFmpeg.
6			*
7			* FFmpeg is free software; you can redistribute it and/or
8			* modify it under the terms of the GNU Lesser General Public
9			* License as published by the Free Software Foundation; either
10			* version 2.1 of the License, or (at your option) any later version.
11			*
12			* FFmpeg is distributed in the hope that it will be useful,
13			* but WITHOUT ANY WARRANTY; without even the implied warranty of
14			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15			* Lesser General Public License for more details.
16			*
17			* You should have received a copy of the GNU Lesser General Public
18			* License along with FFmpeg; if not, write to the Free Software
19			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20			*
21			* Based on cache.c by Michael Niedermayer
22			*/
23
24			#include "libavutil/attributes.h"
25			#include "libavutil/avassert.h"
26			#include "libavutil/avstring.h"
27			#include "libavutil/crc.h"
28			#include "libavutil/error.h"
29			#include "libavutil/hash.h"
30			#include "libavutil/file_open.h"
31			#include "libavutil/mem.h"
32			#include "libavutil/opt.h"
33			#include "libavutil/time.h"
34
35			#include "internal.h"
36			#include "url.h"
37
38			#include <assert.h>
39			#include <errno.h>
40			#include <fcntl.h>
41			#include <inttypes.h>
42			#include <stdatomic.h>
43			#include <string.h>
44			#include <sys/file.h>
45			#include <sys/mman.h>
46			#include <sys/stat.h>
47			#include <unistd.h>
48
49			/**
50			* This hash should be resistant against collision attacks, so that an
51			* attacker could not generate e.g. two different URIs that map to the same
52			* cache file. This requires at least 64 bits of collision resistance in
53			* practice (i.e. 128 bits = 16 bytes of hash size). However, we can be
54			* conservative by computing e.g. a 256 bit hash and storing it inside the
55			* file header for verification.
56			*
57			* Note that due to the way we use atomics, we should avoid zero bytes in
58			* the resulting hash; hence we tweak the input slightly to avoid this.
59			* The resulting loss in hash strength is negligible, since 32 bytes is
60			* already much more than needed.
61			*/
62			#define HASH_METHOD "SHA512/256"
63			#define HASH_SIZE 32
64			#define HEADER_MAGIC MKTAG(u'\xFF', 'S', 'h', '$')
65			#define HEADER_VERSION 3
66
67			/**
68			* Hard watershed of consecutive failed blocks before we give up on the cache
69			* file altogether and assume it's entirely lost to us.
70			**/
71			#define MAX_CORRUPT_BLOCKS 10
72
73		✗	static int hash_uri(uint8_t hash[HASH_SIZE], const char *uri)
74			{
75		✗	struct AVHashContext *ctx = NULL;
76		✗	int ret = av_hash_alloc(&ctx, HASH_METHOD);
77		✗	if (ret < 0)
78		✗	return ret;
79
80		✗	const int16_t version = HEADER_VERSION;
81		✗	av_assert0(av_hash_get_size(ctx) == HASH_SIZE);
82		✗	av_hash_init(ctx);
83		✗	av_hash_update(ctx, (const uint8_t *) &version, sizeof(version));
84		✗	av_hash_update(ctx, (const uint8_t *) uri, strlen(uri));
85		✗	av_hash_final(ctx, hash);
86		✗	av_hash_freep(&ctx);
87
88		✗	for (int i = 0; i < HASH_SIZE; i++)
89		✗	hash[i] = hash[i] ? hash[i] : ~hash[i]; /* prevent zero bytes */
90		✗	return 0;
91			}
92
93			enum BlockState {
94			/* Reserved block state values */
95			BLOCK_NONE = 0, ///< block is not cached
96			BLOCK_PENDING, ///< a thread is currently trying to write this block
97			BLOCK_FAILED, ///< the underlying I/O source failed to read this block
98
99			/**
100			* All other block states represent valid cached blocks, with the value
101			* being the CRC of the block data.
102			*/
103			};
104
105		✗	static uint32_t get_block_crc(const uint8_t *block, size_t block_size)
106			{
107		✗	uint32_t crc = av_crc(av_crc_get_table(AV_CRC_32_IEEE), 0, block, block_size);
108		✗	switch (crc) {
109		✗	case BLOCK_NONE:
110			case BLOCK_FAILED:
111			case BLOCK_PENDING:
112		✗	return ~crc; /* avoid reserved block states */
113		✗	default:
114		✗	return crc;
115			}
116			}
117
118			typedef struct Block {
119			atomic_uint state; /* enum BlockState */
120			} Block;
121
122			typedef struct Spacemap {
123			atomic_uint header_magic;
124			atomic_ushort version;
125			atomic_ushort block_shift;
126			atomic_ullong filesize; /* byte offset of true EOF, or 0 if unknown */
127			atomic_uchar hash[HASH_SIZE]; /* hash of resource URI / filename */
128			char reserved[80];
129
130			Block blocks[];
131			} Spacemap;
132
133			static_assert(offsetof(Spacemap, blocks) == 128, "Spacemap header layout mismatch");
134
135			/* Set to value iff the current value is unset (zero) */
136			#define DEF_SET_ONCE(ctype, atype) \
137			static int set_once_##atype(atomic_##atype *const ptr, const ctype value) \
138			{ \
139			ctype prev = 0; \
140			av_assert1(value != 0); \
141			if (atomic_compare_exchange_strong_explicit( \
142			ptr, &prev, value, memory_order_release, memory_order_relaxed)) \
143			return 1; \
144			else if (prev == value) \
145			return 0; \
146			else \
147			return AVERROR(EINVAL); \
148			}
149
150		✗	DEF_SET_ONCE(unsigned char, uchar)
151		✗	DEF_SET_ONCE(unsigned int, uint)
152		✗	DEF_SET_ONCE(unsigned short, ushort)
153		✗	DEF_SET_ONCE(unsigned long long, ullong)
154
155			typedef struct SharedContext {
156			AVClass *class;
157			URLContext *inner;
158			int64_t inner_pos;
159
160			/* options */
161			char *cache_dir;
162			int block_shift; ///< requested shift; updated on init if it disagrees
163			int read_only;
164			int64_t timeout;
165			int retry_errors;
166			int retry_corrupt;
167			int verify;
168
169			/* misc state */
170			int64_t pos; ///< current logical position
171			uint8_t *tmp_buf;
172			int block_size;
173			int write_err; ///< write error occurred
174			int num_corrupt;
175			int64_t filesize; ///< once known
176
177			/* cache file */
178			uint8_t *cache_data; ///< optional mmap of the cache file
179			char *cache_path;
180			off_t cache_size; ///< size of mapped memory region (for munmap)
181			int fd;
182
183			/* space map */
184			Spacemap *spacemap;
185			char *map_path;
186			off_t map_size;
187			int mapfd;
188
189			/* statistics */
190			int64_t nb_hit;
191			int64_t nb_miss;
192			} SharedContext;
193
194		✗	static int shared_close(URLContext *h)
195			{
196		✗	SharedContext *s = h->priv_data;
197
198		✗	ffurl_close(s->inner);
199		✗	if (s->cache_data)
200		✗	munmap(s->cache_data, s->cache_size);
201		✗	if (s->spacemap)
202		✗	munmap(s->spacemap, s->map_size);
203		✗	if (s->fd != -1)
204		✗	close(s->fd);
205		✗	if (s->mapfd != -1)
206		✗	close(s->mapfd);
207		✗	av_freep(&s->cache_path);
208		✗	av_freep(&s->map_path);
209		✗	av_freep(&s->tmp_buf);
210
211		✗	av_log(h, AV_LOG_DEBUG, "Cache statistics: %"PRId64" hits, %"PRId64" misses\n",
212			s->nb_hit, s->nb_miss);
213		✗	return 0;
214			}
215
216			static int cache_map(URLContext *h, int64_t filesize);
217			static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE]);
218			static int spacemap_grow(URLContext *h, int64_t block);
219
220		✗	static int64_t get_filesize(URLContext *h)
221			{
222		✗	SharedContext *s = h->priv_data;
223		✗	if (!s->filesize) {
224		✗	uint64_t size = atomic_load_explicit(&s->spacemap->filesize, memory_order_relaxed);
225		✗	if (size > INT64_MAX)
226		✗	return AVERROR(EINVAL);
227		✗	else if (size)
228		✗	s->filesize = size;
229			}
230
231		✗	return s->filesize;
232			}
233
234		✗	static int set_filesize(URLContext *h, int64_t new_size)
235			{
236		✗	SharedContext *s = h->priv_data;
237			int ret;
238
239		✗	if (!new_size)
240		✗	return 0;
241
242		✗	ret = set_once_ullong(&s->spacemap->filesize, new_size);
243		✗	if (ret < 0) {
244		✗	av_log(h, AV_LOG_ERROR, "Cached file size mismatch, expected: "
245			"%"PRId64", got: %"PRIu64"!\n", new_size,
246		✗	(uint64_t) atomic_load(&s->spacemap->filesize));
247		✗	return ret;
248		✗	} else if (ret) {
249			/* Opportunistically map the file; this also sets the correct filesize.
250			* Ignore errors as this is not critical to the cache logic. */
251		✗	cache_map(h, new_size);
252			}
253
254		✗	return ret;
255			}
256
257		✗	static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary **options)
258			{
259		✗	SharedContext *s = h->priv_data;
260			int ret;
261
262		✗	if (!s->cache_dir || !s->cache_dir[0]) {
263		✗	av_log(h, AV_LOG_ERROR, "Missing path for shared cache! Specify a "
264			"directory using the -cache_dir option.\n");
265		✗	return AVERROR(EINVAL);
266			}
267
268		✗	s->fd = s->mapfd = -1; /* Set these early for shared_close() failure path */
269
270			/* Open underlying protocol */
271		✗	av_strstart(arg, "shared:", &arg);
272		✗	ret = ffurl_open_whitelist(&s->inner, arg, flags, &h->interrupt_callback,
273			options, h->protocol_whitelist, h->protocol_blacklist, h);
274
275		✗	if (ret < 0)
276		✗	goto fail;
277
278			uint8_t hash[HASH_SIZE];
279		✗	ret = hash_uri(hash, arg);
280		✗	if (ret < 0)
281		✗	goto fail;
282
283			/* 128 bits is enough for collision resistance; we already store the full
284			* hash inside the header for verification */
285			char filename[2 * 16 + 1];
286		✗	ff_data_to_hex(filename, hash, sizeof(filename)/2, 0);
287		✗	s->cache_path = av_asprintf("%s/%s.cache", s->cache_dir, filename);
288		✗	s->map_path = av_asprintf("%s/%s.spacemap", s->cache_dir, filename);
289		✗	if (!s->cache_path || !s->map_path) {
290		✗	ret = AVERROR(ENOMEM);
291		✗	goto fail;
292			}
293
294		✗	av_log(h, AV_LOG_VERBOSE, "Opening cache file '%s' for URI: '%s'\n",
295		✗	s->cache_path, s->inner->filename);
296
297		✗	s->fd = avpriv_open(s->cache_path, O_RDWR | O_CREAT, 0660);
298		✗	s->mapfd = s->fd >= 0 ? avpriv_open(s->map_path, O_RDWR | O_CREAT, 0660) : -1;
299		✗	if (s->fd < 0 || s->mapfd < 0) {
300		✗	ret = AVERROR(errno);
301		✗	av_log(h, AV_LOG_ERROR, "Failed to open '%s': %s\n",
302		✗	s->fd < 0 ? s->cache_path : s->map_path, av_err2str(ret));
303		✗	goto fail;
304			}
305
306		✗	ret = spacemap_init(h, hash);
307		✗	if (ret < 0)
308		✗	goto fail;
309
310			/* s->block_shift is fully settled after spacemap_init() */
311		✗	s->block_size = 1 << s->block_shift;
312
313		✗	int64_t filesize = get_filesize(h);
314		✗	if (filesize < 0) {
315		✗	ret = (int) filesize;
316		✗	goto fail;
317		✗	} else if (!filesize) {
318			/* Filesize is not yet known, try to get it from the underlying URL */
319		✗	filesize = ffurl_size(s->inner);
320		✗	if (filesize < 0 && filesize != AVERROR(ENOSYS)) {
321		✗	ret = (int) filesize;
322		✗	goto fail;
323		✗	} else if (filesize > 0) {
324		✗	ret = set_filesize(h, filesize);
325		✗	if (ret < 0)
326		✗	goto fail;
327			}
328			}
329
330		✗	if (filesize > 0) {
331		✗	int64_t last_pos = filesize - 1;
332		✗	int64_t last_block = last_pos >> s->block_shift;
333		✗	ret = spacemap_grow(h, last_block);
334		✗	if (ret < 0)
335		✗	goto fail;
336
337			/* If filesize is known, we can directly mmap() the cache file */
338		✗	ret = cache_map(h, filesize);
339		✗	if (ret < 0) {
340		✗	av_log(h, AV_LOG_WARNING, "Failed to map cache file: %s. Falling "
341		✗	"back to normal read/write\n", av_err2str(ret));
342		✗	ret = 0;
343			}
344			}
345
346			/* Temporary buffer needed for pread/pwrite() fallback */
347		✗	s->tmp_buf = av_malloc(s->block_size);
348		✗	if (!s->tmp_buf) {
349		✗	ret = AVERROR(ENOMEM);
350		✗	goto fail;
351			}
352
353		✗	h->max_packet_size = s->block_size;
354		✗	h->min_packet_size = s->block_size;
355		✗	ret = 0;
356
357		✗	fail:
358		✗	if (ret < 0)
359		✗	shared_close(h);
360		✗	return ret;
361			}
362
363		✗	static int cache_map(URLContext *h, int64_t filesize)
364			{
365		✗	SharedContext *s = h->priv_data;
366		✗	if (s->cache_size >= filesize || filesize > SIZE_MAX)
367		✗	return 0;
368
369		✗	if (s->cache_data) {
370		✗	munmap(s->cache_data, s->cache_size);
371		✗	s->cache_data = NULL;
372		✗	s->cache_size = 0;
373			}
374
375			struct stat st;
376		✗	int ret = fstat(s->fd, &st);
377		✗	if (ret < 0)
378		✗	return AVERROR(errno);
379
380		✗	if (st.st_size != filesize) {
381			/* Ensure the file size is correct before mapping; this can happen if
382			* another process wrote the correct filesize to the header but
383			* crashed right before actually successfully resizing the file. */
384		✗	ret = ftruncate(s->fd, filesize);
385		✗	if (ret < 0)
386		✗	return AVERROR(errno);
387			}
388
389		✗	s->cache_data = mmap(NULL, filesize, PROT_READ | PROT_WRITE, MAP_SHARED, s->fd, 0);
390		✗	if (s->cache_data == MAP_FAILED) {
391		✗	s->cache_data = NULL;
392		✗	return AVERROR(errno);
393			}
394
395		✗	s->cache_size = filesize;
396		✗	return 0;
397			}
398
399		✗	static int spacemap_remap(URLContext *h, size_t map_size)
400			{
401		✗	SharedContext *s = h->priv_data;
402		✗	int ret, did_grow = 0, locked = 0;
403		✗	if (map_size <= s->map_size)
404		✗	return 0;
405
406			/* Opportunistically get current filesize before attempting to lock */
407			struct stat st;
408		✗	ret = fstat(s->mapfd, &st);
409		✗	if (ret < 0) {
410		✗	ret = AVERROR(errno);
411		✗	goto fail;
412			}
413
414		✗	if (st.st_size >= map_size)
415		✗	goto skip_resize;
416
417			/* Lock the spacemap to ensure nobody else is currently resizing it */
418		✗	ret = flock(s->mapfd, LOCK_EX);
419		✗	if (ret < 0) {
420		✗	ret = AVERROR(errno);
421		✗	goto fail;
422			}
423		✗	locked = 1;
424
425			/* Refresh filesize after acquiring the lock */
426		✗	ret = fstat(s->mapfd, &st);
427		✗	if (ret < 0) {
428		✗	ret = AVERROR(errno);
429		✗	goto fail;
430			}
431
432		✗	if (st.st_size >= map_size)
433		✗	goto skip_resize;
434
435		✗	ret = ftruncate(s->mapfd, map_size);
436		✗	if (ret < 0) {
437		✗	ret = AVERROR(errno);
438		✗	goto fail;
439			}
440		✗	st.st_size = map_size;
441		✗	did_grow = 1;
442
443		✗	skip_resize:
444		✗	if (s->spacemap)
445		✗	munmap(s->spacemap, s->map_size);
446		✗	s->map_size = st.st_size;
447		✗	s->spacemap = mmap(NULL, s->map_size, PROT_READ | PROT_WRITE, MAP_SHARED, s->mapfd, 0);
448		✗	if (s->spacemap == MAP_FAILED) {
449		✗	s->spacemap = NULL; /* for munmap check */
450		✗	s->map_size = 0;
451		✗	ret = AVERROR(errno);
452		✗	goto fail;
453			}
454
455		✗	if (locked) {
456		✗	flock(s->mapfd, LOCK_UN);
457		✗	locked = 0;
458			}
459
460		✗	return did_grow;
461
462		✗	fail:
463		✗	if (locked)
464		✗	flock(s->mapfd, LOCK_UN);
465		✗	av_log(h, AV_LOG_ERROR, "Failed to resize space map: %s\n", av_err2str(ret));
466		✗	return ret;
467			}
468
469		✗	static int spacemap_grow(URLContext *h, int64_t block)
470			{
471		✗	SharedContext *s = h->priv_data;
472		✗	int64_t num_blocks = block + 1;
473		✗	size_t map_bytes = sizeof(Spacemap) + num_blocks * sizeof(Block);
474
475			/* When streaming files without known size, round up the number of blocks
476			* to the nearest multiple of the block size to reduce the rate of resizes */
477		✗	int64_t filesize = get_filesize(h);
478		✗	if (filesize < 0)
479		✗	return (int) filesize;
480		✗	else if (!filesize) {
481		✗	av_assert0(s->block_size > 0);
482		✗	map_bytes = FFALIGN(map_bytes, (int64_t) s->block_size);
483			}
484
485		✗	if (map_bytes < num_blocks)
486		✗	return AVERROR(EINVAL); /* overflow */
487
488		✗	const off_t old_size = s->map_size;
489		✗	int ret = spacemap_remap(h, map_bytes);
490		✗	if (ret < 0)
491		✗	return ret;
492
493			/* Report new size after successful grow */
494		✗	if (s->map_size > old_size) {
495		✗	num_blocks = (s->map_size - sizeof(Spacemap)) / sizeof(Block);
496		✗	av_log(h, AV_LOG_DEBUG,
497			"%s %zu bytes, capacity: %"PRId64" blocks = %"PRId64" MB\n",
498			ret ? "Resized spacemap to" : "Mapped spacemap with",
499		✗	(size_t) s->map_size, num_blocks,
500		✗	(num_blocks * (int64_t) s->block_size) >> 20);
501			}
502		✗	return 0;
503			}
504
505		✗	static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE])
506			{
507		✗	SharedContext *s = h->priv_data;
508			int ret;
509
510		✗	ret = spacemap_remap(h, sizeof(Spacemap));
511		✗	if (ret < 0)
512		✗	return ret;
513
514		✗	if ((ret = set_once_uint(&s->spacemap->header_magic, HEADER_MAGIC)) < 0 ||
515		✗	(ret = set_once_ushort(&s->spacemap->version, HEADER_VERSION)) < 0)
516			{
517		✗	av_log(h, AV_LOG_ERROR, "Shared cache spacemap header mismatch!\n");
518		✗	av_log(h, AV_LOG_ERROR, " Expected magic: 0x%X, version: %d\n",
519			HEADER_MAGIC, HEADER_VERSION);
520		✗	av_log(h, AV_LOG_ERROR, " Got magic: 0x%X, version: %d\n",
521		✗	atomic_load(&s->spacemap->header_magic),
522		✗	atomic_load(&s->spacemap->version));
523		✗	return ret;
524			}
525
526		✗	ret = set_once_ushort(&s->spacemap->block_shift, s->block_shift);
527		✗	if (ret < 0) {
528		✗	const int shift = atomic_load(&s->spacemap->block_shift);
529		✗	av_log(h, AV_LOG_WARNING, "Shared cache uses block shift %d, "
530			"but requested block shift is %d.\n", shift, s->block_shift);
531		✗	if (shift < 9 || shift > 30) {
532		✗	av_log(h, AV_LOG_ERROR, "Invalid block shift %d in cache file!\n", shift);
533		✗	return AVERROR(EINVAL);
534			}
535		✗	s->block_shift = shift;
536			}
537
538		✗	for (int i = 0; i < HASH_SIZE; i++) {
539		✗	ret = set_once_uchar(&s->spacemap->hash[i], hash[i]);
540		✗	if (ret < 0) {
541		✗	av_log(h, AV_LOG_ERROR, "Shared cache spacemap hash mismatch!\n");
542			char hash_hex[2 * HASH_SIZE + 1];
543		✗	ff_data_to_hex(hash_hex, hash, HASH_SIZE, 0);
544		✗	av_log(h, AV_LOG_ERROR, " Expected hash: %s\n", hash_hex);
545			uint8_t hash2[HASH_SIZE];
546		✗	for (int j = 0; j < HASH_SIZE; ++j)
547		✗	hash2[j] = atomic_load_explicit(&s->spacemap->hash[j], memory_order_relaxed);
548		✗	ff_data_to_hex(hash_hex, hash2, HASH_SIZE, 0);
549		✗	av_log(h, AV_LOG_ERROR, " Got hash: %s\n", hash_hex);
550		✗	return ret;
551			}
552			}
553
554		✗	if (ret) /* set_once() return 1 if this is the first time setting the value */
555		✗	av_log(h, AV_LOG_DEBUG, "Initialized new cache spacemap.\n");
556
557		✗	return ret;
558			}
559
560		✗	static int read_cache(SharedContext *s, uint8_t *buf, size_t size, off_t offset)
561			{
562		✗	if (s->cache_data) {
563			av_assert1(offset + size <= s->cache_size);
564		✗	memcpy(buf, s->cache_data + offset, size);
565		✗	return 0;
566			}
567
568		✗	while (size) {
569		✗	ssize_t ret = pread(s->fd, buf, size, offset);
570		✗	if (ret <= 0)
571		✗	return ret ? AVERROR(errno) : AVERROR_EOF;
572		✗	buf += ret;
573		✗	offset += ret;
574		✗	size -= ret;
575			}
576
577		✗	return 0;
578			}
579
580		✗	static int write_cache(SharedContext *s, const uint8_t *buf, size_t size, off_t offset)
581			{
582		✗	if (s->cache_data) {
583			av_assert1(offset + size <= s->cache_size);
584		✗	memcpy(s->cache_data + offset, buf, size);
585		✗	return 0;
586			}
587
588		✗	while (size) {
589		✗	ssize_t ret = pwrite(s->fd, buf, size, offset);
590		✗	if (ret <= 0)
591		✗	return ret ? AVERROR(errno) : AVERROR(EIO);
592		✗	buf += ret;
593		✗	offset += ret;
594		✗	size -= ret;
595			}
596
597		✗	return 0;
598			}
599
600		✗	static int clamp_size(URLContext *h, int size, int64_t pos, int64_t filesize)
601			{
602		✗	if (!filesize)
603		✗	return size;
604		✗	else if (pos > filesize)
605		✗	return 0;
606			else
607		✗	return FFMIN(filesize - pos, size);
608			}
609
610		✗	static int shared_read(URLContext *h, unsigned char *buf, int size)
611			{
612		✗	SharedContext *s = h->priv_data;
613			uint8_t *tmp;
614			int ret;
615		✗	if (!s->spacemap)
616		✗	return AVERROR(EIO);
617
618		✗	if (size <= 0)
619		✗	return 0;
620
621		✗	int64_t filesize = get_filesize(h);
622		✗	if (filesize < 0)
623		✗	return (int) filesize;
624
625		✗	size = clamp_size(h, size, s->pos, filesize);
626		✗	if (size <= 0)
627		✗	return AVERROR_EOF;
628
629		✗	const int64_t block_id = s->pos >> s->block_shift;
630		✗	const int64_t offset = s->pos & (s->block_size - 1);
631		✗	const int64_t block_pos = block_id * s->block_size;
632		✗	int block_size = clamp_size(h, s->block_size, block_pos, filesize);
633		✗	ret = spacemap_grow(h, block_id);
634		✗	if (ret < 0)
635		✗	return ret;
636
637		✗	Block *const block = &s->spacemap->blocks[block_id];
638		✗	unsigned state = atomic_load_explicit(&block->state, memory_order_acquire);
639		✗	int64_t pending_since = 0;
640		✗	int verify_read = 0, acquired = 0;
641
642		✗	retry:
643		✗	switch (state) {
644		✗	default:
645		✗	if (s->num_corrupt >= MAX_CORRUPT_BLOCKS)
646		✗	goto read_block; /* assume broken cache file */
647
648			/* filesize may have become known in the meantime */
649		✗	filesize = get_filesize(h);
650		✗	if (filesize < 0)
651		✗	return (int) filesize;
652
653			/* We always need to read the entire block to verify integrity */
654		✗	block_size = clamp_size(h, block_size, block_pos, filesize);
655		✗	if (s->cache_data) {
656			av_assert1(block_pos + block_size <= s->cache_size);
657		✗	tmp = s->cache_data + block_pos;
658			} else {
659		✗	tmp = s->tmp_buf;
660		✗	ret = read_cache(s, tmp, block_size, block_pos);
661		✗	if (ret < 0) {
662		✗	av_log(h, AV_LOG_ERROR, "Failed to read from cache file: %s\n", av_err2str(ret));
663		✗	if (ret == AVERROR_EOF) { /* e.g. cache appears truncated? */
664		✗	if (s->retry_corrupt) {
665		✗	s->num_corrupt++;
666		✗	goto read_block;
667			}
668		✗	ret = AVERROR(EIO); /* don't propagate EOF to caller */
669			}
670		✗	return ret;
671			}
672			}
673
674		✗	uint32_t crc = get_block_crc(tmp, block_size);
675		✗	if (crc != state) {
676		✗	av_log(h, AV_LOG_ERROR, "Cache corruption detected for block 0x%"PRIx64" at "
677			"offset 0x%"PRIx64": expected CRC: 0x%08X, got: 0x%08X\n",
678			block_id, block_pos, state, crc);
679		✗	if (s->retry_corrupt) {
680		✗	s->num_corrupt++;
681		✗	goto read_block;
682			}
683		✗	return AVERROR(EIO);
684			} else
685		✗	s->num_corrupt = 0; /* reset corrupt block count on success */
686
687		✗	tmp += (ptrdiff_t) offset;
688		✗	size = FFMIN(size, block_size - offset);
689		✗	if (size <= 0)
690		✗	return AVERROR_EOF;
691		✗	if (s->verify) {
692		✗	verify_read = 1;
693		✗	break; /* fall through to the cache miss logic */
694			}
695
696		✗	memcpy(buf, tmp, size);
697		✗	s->nb_hit++;
698		✗	s->pos += size;
699		✗	return size;
700
701		✗	case BLOCK_FAILED:
702		✗	if (s->retry_errors)
703		✗	goto read_block;
704		✗	return AVERROR(EIO);
705
706		✗	read_block:
707		✗	if (s->num_corrupt == MAX_CORRUPT_BLOCKS) {
708		✗	av_log(h, AV_LOG_ERROR, "Too many consecutive corrupt blocks; "
709			"assuming cache file is completely broken.\n");
710		✗	s->num_corrupt++; /* silence this log on subsequent reads */
711			}
712			av_fallthrough;
713
714			case BLOCK_NONE:
715		✗	if (s->read_only || s->write_err)
716			break; /* don't mark block as pending */
717		✗	if (atomic_compare_exchange_strong_explicit(&block->state, &state,
718			BLOCK_PENDING,
719			memory_order_acquire,
720			memory_order_acquire))
721			{
722			/* Acquired pending state, proceed to fetch the block */
723		✗	acquired = 1;
724		✗	state = BLOCK_PENDING;
725		✗	break;
726			}
727			/* CAS failed, another thread changed the state; reload it */
728		✗	goto retry;
729
730		✗	case BLOCK_PENDING:
731			/* Another thread is busy fetching this block, wait for it to finish */
732		✗	if (!s->timeout) {
733		✗	break; /* no timeout requested, immediately race to fetch block */
734		✗	} else if (pending_since) {
735		✗	int64_t new = av_gettime_relative();
736		✗	if (new - pending_since >= s->timeout)
737		✗	break; /* timeout expired, try to fetch the block ourselves */
738			} else {
739		✗	pending_since = av_gettime_relative();
740			}
741
742		✗	if (h->flags & AVIO_FLAG_NONBLOCK)
743		✗	return AVERROR(EAGAIN);
744
745			/* Make sure we try a few times before giving up */
746		✗	av_usleep(FFMIN(s->timeout >> 4, 10000));
747		✗	if (ff_check_interrupt(&h->interrupt_callback))
748		✗	return AVERROR_EXIT;
749
750		✗	state = atomic_load_explicit(&block->state, memory_order_acquire);
751		✗	goto retry;
752			}
753
754			/* Release pending state on failure to avoid stalling other threads */
755			#define RELEASE_PENDING(block, state) \
756			do { \
757			if (acquired) { \
758			av_assert1(state == BLOCK_PENDING); \
759			atomic_compare_exchange_strong_explicit( \
760			&block->state, &state, BLOCK_NONE, memory_order_relaxed, \
761			memory_order_relaxed); \
762			} \
763			} while (0)
764
765			/* Cache miss, fetch this block from underlying protocol */
766		✗	s->nb_miss++;
767
768		✗	const int read_only = s->read_only || s->write_err || verify_read;
769		✗	int64_t inner_pos = read_only ? s->pos : block_pos;
770		✗	if (s->inner_pos != inner_pos) {
771		✗	inner_pos = ffurl_seek(s->inner, inner_pos, SEEK_SET);
772		✗	if (inner_pos < 0) {
773		✗	av_log(h, AV_LOG_ERROR, "Failed to seek underlying protocol: %s\n",
774		✗	av_err2str(inner_pos));
775		✗	RELEASE_PENDING(block, state);
776		✗	return inner_pos;
777			}
778
779		✗	av_log(h, AV_LOG_DEBUG, "Inner seek to 0x%"PRIx64"\n", inner_pos);
780		✗	s->inner_pos = inner_pos;
781			}
782
783		✗	if (read_only) {
784			/* Directly defer to the underlying protocol */
785		✗	ret = ffurl_read(s->inner, buf, size);
786		✗	if (ret < 0) {
787			av_assert1(!acquired);
788		✗	return ret;
789			} else {
790		✗	s->inner_pos = inner_pos + ret;
791			}
792
793			/* Verify the read data against the cached data if requested */
794		✗	if (verify_read && memcmp(buf, tmp, ret)) {
795		✗	av_log(h, AV_LOG_ERROR, "Cache verification failed for %d bytes "
796			"in block 0x%"PRIx64" at offset 0x%"PRIx64" + %"PRId64"!\n",
797			ret, block_id, block_pos, offset);
798		✗	return AVERROR(EIO);
799			}
800
801		✗	s->pos = s->inner_pos;
802		✗	return ret;
803			}
804
805		✗	int write_back = 1;
806		✗	if (s->cache_data && acquired) {
807			/* Read directly into memory mapped cache file */
808		✗	tmp = s->cache_data + block_pos;
809		✗	write_back = 0;
810		✗	} else if (size >= block_size && !offset) {
811			/* Read directly into output buffer if aligned and large enough */
812		✗	tmp = buf;
813			} else {
814			/* Read into temporary buffer and copy later */
815		✗	tmp = s->tmp_buf;
816			}
817
818			/* Try and fetch the entire block */
819		✗	av_assert0(inner_pos == block_pos);
820		✗	int bytes_read = 0;
821		✗	while (bytes_read < block_size) {
822		✗	ret = ffurl_read(s->inner, &tmp[bytes_read], block_size - bytes_read);
823		✗	if (!ret || ret == AVERROR_EOF)
824			break;
825		✗	else if (ret < 0) {
826		✗	av_log(h, AV_LOG_ERROR, "Failed to read block 0x%"PRIx64": %s\n",
827		✗	block_id, av_err2str(ret));
828		✗	if (ret == AVERROR(EAGAIN) || ret == AVERROR_EXIT) {
829		✗	RELEASE_PENDING(block, state);
830		✗	return ret; /* transient error, allow retries */
831			}
832
833			/* Try to mark block as failed; ignore errors - any mismatch
834			* here will mean that either another thread already marked it
835			* as failed, or successfully cached it in the meantime */
836		✗	atomic_compare_exchange_strong_explicit(&block->state, &state,
837			BLOCK_FAILED,
838			memory_order_relaxed,
839			memory_order_relaxed);
840		✗	return ret;
841			}
842
843		✗	bytes_read += ret;
844		✗	s->inner_pos += ret;
845			}
846
847		✗	if (bytes_read < block_size) {
848			/* Learned location of true EOF, update filesize */
849		✗	ret = set_filesize(h, inner_pos + bytes_read);
850		✗	if (ret < 0) {
851		✗	RELEASE_PENDING(block, state);
852		✗	return ret;
853			}
854			}
855
856		✗	if (bytes_read > 0) {
857		✗	ret = write_back ? write_cache(s, tmp, bytes_read, block_pos) : 0;
858		✗	if (ret < 0) {
859		✗	if (ret != AVERROR(EINTR)) {
860		✗	av_log(h, AV_LOG_ERROR, "Failed to write to cache file: %s\n",
861		✗	av_err2str(ret));
862		✗	s->write_err = 1;
863			}
864		✗	RELEASE_PENDING(block, state);
865			} else {
866		✗	uint32_t crc = get_block_crc(tmp, bytes_read);
867		✗	av_log(h, AV_LOG_TRACE, "Cached %d bytes to block 0x%"PRIx64" at "
868			"offset 0x%"PRIx64", CRC 0x%08X\n", bytes_read, block_id,
869			block_pos, crc);
870		✗	atomic_store_explicit(&block->state, crc, memory_order_release);
871			}
872			} else {
873		✗	RELEASE_PENDING(block, state);
874		✗	return AVERROR_EOF;
875			}
876
877		✗	size = FFMIN(bytes_read - offset, size);
878		✗	if (size <= 0)
879		✗	return AVERROR_EOF;
880		✗	if (tmp != buf)
881		✗	memcpy(buf, &tmp[offset], size);
882		✗	s->pos += size;
883		✗	return size;
884			}
885
886		✗	static int64_t shared_seek(URLContext *h, int64_t pos, int whence)
887			{
888		✗	SharedContext *s = h->priv_data;
889			int64_t res;
890		✗	if (!s->spacemap)
891		✗	return AVERROR(EIO);
892
893		✗	const int64_t filesize = get_filesize(h);
894		✗	if (filesize < 0)
895		✗	return filesize;
896
897		✗	switch (whence) {
898		✗	case AVSEEK_SIZE:
899		✗	if (filesize)
900		✗	return filesize;
901		✗	res = ffurl_seek(s->inner, pos, whence);
902		✗	if (res > 0) {
903		✗	if (set_filesize(h, res) < 0)
904		✗	return AVERROR(EINVAL);
905			}
906		✗	return res;
907		✗	case SEEK_SET:
908		✗	break;
909		✗	case SEEK_CUR:
910		✗	pos += s->pos;
911		✗	break;
912		✗	case SEEK_END:
913		✗	if (filesize) {
914		✗	pos += filesize;
915		✗	break;
916			}
917			/* Defer to underlying protocol if filesize is unknown */
918		✗	res = ffurl_seek(s->inner, pos, whence);
919		✗	if (res < 0)
920		✗	return res;
921			/* Opportunistically update known filesize */
922		✗	if (set_filesize(h, res - pos) < 0)
923		✗	return AVERROR(EINVAL);
924		✗	av_log(h, AV_LOG_DEBUG, "Inner seek to 0x%"PRIx64"\n", res);
925		✗	return s->pos = s->inner_pos = res;
926		✗	default:
927		✗	return AVERROR(EINVAL);
928			}
929
930		✗	if (pos < 0)
931		✗	return AVERROR(EINVAL);
932
933		✗	av_log(h, AV_LOG_DEBUG, "Virtual seek to 0x%"PRIx64"\n", pos);
934		✗	return s->pos = pos;
935			}
936
937		✗	static int shared_get_file_handle(URLContext *h)
938			{
939		✗	SharedContext *s = h->priv_data;
940		✗	return ffurl_get_file_handle(s->inner);
941			}
942
943		✗	static int shared_get_short_seek(URLContext *h)
944			{
945		✗	SharedContext *s = h->priv_data;
946		✗	int ret = ffurl_get_short_seek(s->inner);
947		✗	return ret > 0 ? FFMAX(ret, s->block_size) : s->block_size;
948			}
949
950			#define OFFSET(x) offsetof(SharedContext, x)
951			#define D AV_OPT_FLAG_DECODING_PARAM
952
953			static const AVOption options[] = {
954			{ "cache_dir", "Directory path for shared file cache", OFFSET(cache_dir), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D },
955			{ "block_shift", "Set the base 2 logarithm of the block size", OFFSET(block_shift), AV_OPT_TYPE_INT, {.i64 = 15}, 9, 30, .flags = D },
956			{ "read_only", "Don't write data to the cache, only read from it", OFFSET(read_only), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
957			{ "cache_verify", "Verify correctness of the cache against the source", OFFSET(verify), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
958			{ "cache_timeout", "Time in us to wait before re-fetching pending blocks", OFFSET(timeout), AV_OPT_TYPE_INT64, {.i64 = 10000}, 0, INT64_MAX, .flags = D },
959			{ "retry_errors", "Re-request blocks even if they previously failed", OFFSET(retry_errors), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
960			{ "retry_corrupt", "Re-request blocks that fail the CRC check", OFFSET(retry_corrupt), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
961			{0},
962			};
963
964			static const AVClass shared_context_class = {
965			.class_name = "shared",
966			.item_name = av_default_item_name,
967			.option = options,
968			.version = LIBAVUTIL_VERSION_INT,
969			};
970
971			const URLProtocol ff_shared_protocol = {
972			.name = "shared",
973			.url_open2 = shared_open,
974			.url_read = shared_read,
975			.url_seek = shared_seek,
976			.url_close = shared_close,
977			.url_get_file_handle = shared_get_file_handle,
978			.url_get_short_seek = shared_get_short_seek,
979			.priv_data_size = sizeof(SharedContext),
980			.priv_data_class = &shared_context_class,
981			};
982