FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Shared file cache protocol.
3			* Copyright (c) 2026 Niklas Haas
4			*
5			* This file is part of FFmpeg.
6			*
7			* FFmpeg is free software; you can redistribute it and/or
8			* modify it under the terms of the GNU Lesser General Public
9			* License as published by the Free Software Foundation; either
10			* version 2.1 of the License, or (at your option) any later version.
11			*
12			* FFmpeg is distributed in the hope that it will be useful,
13			* but WITHOUT ANY WARRANTY; without even the implied warranty of
14			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15			* Lesser General Public License for more details.
16			*
17			* You should have received a copy of the GNU Lesser General Public
18			* License along with FFmpeg; if not, write to the Free Software
19			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20			*
21			* Based on cache.c by Michael Niedermayer
22			*/
23
24			#include "libavutil/attributes.h"
25			#include "libavutil/avassert.h"
26			#include "libavutil/avstring.h"
27			#include "libavutil/crc.h"
28			#include "libavutil/error.h"
29			#include "libavutil/file.h"
30			#include "libavutil/hash.h"
31			#include "libavutil/file_open.h"
32			#include "libavutil/mem.h"
33			#include "libavutil/opt.h"
34			#include "libavutil/time.h"
35
36			#include "internal.h"
37			#include "os_support.h"
38			#include "url.h"
39
40			#include <assert.h>
41			#include <errno.h>
42			#include <fcntl.h>
43			#include <inttypes.h>
44			#include <stdatomic.h>
45			#include <string.h>
46			#include <sys/stat.h>
47			#if HAVE_UNISTD_H
48			#include <unistd.h>
49			#endif
50			#ifndef _WIN32
51			#include <sys/file.h>
52			#endif
53
54			/**
55			* This hash should be resistant against collision attacks, so that an
56			* attacker could not generate e.g. two different URIs that map to the same
57			* cache file. This requires at least 64 bits of collision resistance in
58			* practice (i.e. 128 bits = 16 bytes of hash size). However, we can be
59			* conservative by computing e.g. a 256 bit hash and storing it inside the
60			* file header for verification.
61			*
62			* Note that due to the way we use atomics, we should avoid zero bytes in
63			* the resulting hash; hence we tweak the input slightly to avoid this.
64			* The resulting loss in hash strength is negligible, since 32 bytes is
65			* already much more than needed.
66			*/
67			#define HASH_METHOD "SHA512/256"
68			#define HASH_SIZE 32
69			#define HEADER_MAGIC MKTAG(u'\xFF', 'S', 'h', '$')
70			#define HEADER_VERSION 3
71
72			/**
73			* Hard watershed of consecutive failed blocks before we give up on the cache
74			* file altogether and assume it's entirely lost to us.
75			**/
76			#define MAX_CORRUPT_BLOCKS 10
77
78		✗	static int hash_uri(uint8_t hash[HASH_SIZE], const char *uri)
79			{
80		✗	struct AVHashContext *ctx = NULL;
81		✗	int ret = av_hash_alloc(&ctx, HASH_METHOD);
82		✗	if (ret < 0)
83		✗	return ret;
84
85		✗	const int16_t version = HEADER_VERSION;
86		✗	av_assert0(av_hash_get_size(ctx) == HASH_SIZE);
87		✗	av_hash_init(ctx);
88		✗	av_hash_update(ctx, (const uint8_t *) &version, sizeof(version));
89		✗	av_hash_update(ctx, (const uint8_t *) uri, strlen(uri));
90		✗	av_hash_final(ctx, hash);
91		✗	av_hash_freep(&ctx);
92
93		✗	for (int i = 0; i < HASH_SIZE; i++)
94		✗	hash[i] = hash[i] ? hash[i] : ~hash[i]; /* prevent zero bytes */
95		✗	return 0;
96			}
97
98			enum BlockState {
99			/* Reserved block state values */
100			BLOCK_NONE = 0, ///< block is not cached
101			BLOCK_PENDING, ///< a thread is currently trying to write this block
102			BLOCK_FAILED, ///< the underlying I/O source failed to read this block
103
104			/**
105			* All other block states represent valid cached blocks, with the value
106			* being the CRC of the block data.
107			*/
108			};
109
110		✗	static uint32_t get_block_crc(const uint8_t *block, size_t block_size)
111			{
112		✗	uint32_t crc = av_crc(av_crc_get_table(AV_CRC_32_IEEE), 0, block, block_size);
113		✗	switch (crc) {
114		✗	case BLOCK_NONE:
115			case BLOCK_FAILED:
116			case BLOCK_PENDING:
117		✗	return ~crc; /* avoid reserved block states */
118		✗	default:
119		✗	return crc;
120			}
121			}
122
123			typedef struct Block {
124			atomic_uint state; /* enum BlockState */
125			} Block;
126
127			typedef struct Spacemap {
128			atomic_uint header_magic;
129			atomic_ushort version;
130			atomic_ushort block_shift;
131			atomic_ullong filesize; /* byte offset of true EOF, or 0 if unknown */
132			atomic_uchar hash[HASH_SIZE]; /* hash of resource URI / filename */
133			atomic_ullong blocks_cached; /* (lower bound on) the number of blocks cached */
134			char reserved[72];
135
136			Block blocks[];
137			} Spacemap;
138
139			static_assert(offsetof(Spacemap, blocks) == 128, "Spacemap header layout mismatch");
140
141			/* Set to value iff the current value is unset (zero) */
142			#define DEF_SET_ONCE(ctype, atype) \
143			static int set_once_##atype(atomic_##atype *const ptr, const ctype value) \
144			{ \
145			ctype prev = 0; \
146			av_assert1(value != 0); \
147			if (atomic_compare_exchange_strong_explicit( \
148			ptr, &prev, value, memory_order_release, memory_order_relaxed)) \
149			return 1; \
150			else if (prev == value) \
151			return 0; \
152			else \
153			return AVERROR(EINVAL); \
154			}
155
156		✗	DEF_SET_ONCE(unsigned char, uchar)
157		✗	DEF_SET_ONCE(unsigned int, uint)
158		✗	DEF_SET_ONCE(unsigned short, ushort)
159		✗	DEF_SET_ONCE(unsigned long long, ullong)
160
161			typedef struct SharedContext {
162			AVClass *class;
163			URLContext *inner;
164			int64_t inner_pos;
165
166			/* options */
167			char *cache_dir;
168			char *metadata_dir;
169			int block_shift; ///< requested shift; updated on init if it disagrees
170			int read_only;
171			int64_t timeout;
172			int ignore_errors;
173			int retry_errors;
174			int retry_corrupt;
175			int verify;
176			int64_t cache_size_max;
177
178			/* misc state */
179			int64_t pos; ///< current logical position
180			uint8_t *tmp_buf;
181			int block_size;
182			int write_err; ///< write error occurred
183			int num_corrupt;
184			int64_t filesize; ///< once known
185			int64_t blocks_max; ///< maximum number of blocks to cache
186
187			/* cache file */
188			char *cache_path;
189			int fd;
190
191			/* space map */
192			Spacemap *spacemap;
193			char *map_path;
194			off_t map_size;
195			int mapfd;
196
197			/* statistics */
198			int64_t nb_hit;
199			int64_t nb_miss;
200			} SharedContext;
201
202		✗	static int shared_close(URLContext *h)
203			{
204		✗	SharedContext *s = h->priv_data;
205
206		✗	ffurl_close(s->inner);
207		✗	av_file_unmap_shared(s->spacemap, s->map_size);
208		✗	if (s->fd != -1)
209		✗	close(s->fd);
210		✗	if (s->mapfd != -1)
211		✗	close(s->mapfd);
212		✗	av_freep(&s->cache_path);
213		✗	av_freep(&s->map_path);
214		✗	av_freep(&s->tmp_buf);
215
216		✗	av_log(h, AV_LOG_DEBUG, "Cache statistics: %"PRId64" hits, %"PRId64" misses\n",
217			s->nb_hit, s->nb_miss);
218		✗	return 0;
219			}
220
221			static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE]);
222			static int spacemap_grow(URLContext *h, int64_t block);
223
224		✗	static int64_t get_filesize(URLContext *h)
225			{
226		✗	SharedContext *s = h->priv_data;
227		✗	if (!s->filesize) {
228		✗	uint64_t size = atomic_load_explicit(&s->spacemap->filesize, memory_order_relaxed);
229		✗	if (size > INT64_MAX)
230		✗	return AVERROR(EINVAL);
231		✗	else if (size)
232		✗	s->filesize = size;
233			}
234
235		✗	return s->filesize;
236			}
237
238		✗	static int set_filesize(URLContext *h, int64_t new_size)
239			{
240		✗	SharedContext *s = h->priv_data;
241			int ret;
242
243		✗	if (!new_size)
244		✗	return 0;
245
246		✗	ret = set_once_ullong(&s->spacemap->filesize, new_size);
247		✗	if (ret < 0) {
248		✗	av_log(h, AV_LOG_ERROR, "Cached file size mismatch, expected: "
249			"%"PRId64", got: %"PRIu64"!\n", new_size,
250		✗	(uint64_t) atomic_load(&s->spacemap->filesize));
251		✗	return ret;
252			}
253
254		✗	return ret;
255			}
256
257		✗	static int is_ignorable_error(int64_t err)
258			{
259		✗	switch (err) {
260		✗	case AVERROR_EXIT:
261			case AVERROR_EOF:
262			case AVERROR_BUG:
263			case AVERROR(EAGAIN):
264			case AVERROR(ENOSYS):
265			case AVERROR(EINVAL):
266		✗	return 0;
267		✗	default:
268		✗	return err < 0;
269			}
270			}
271
272		✗	static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary **options)
273			{
274		✗	SharedContext *s = h->priv_data;
275			int ret;
276
277		✗	if (!s->cache_dir || !s->cache_dir[0]) {
278		✗	av_log(h, AV_LOG_ERROR, "Missing path for shared cache! Specify a "
279			"directory using the -cache_dir option.\n");
280		✗	return AVERROR(EINVAL);
281			}
282
283		✗	const char *spacemap_dir = s->cache_dir;
284		✗	if (s->metadata_dir && s->metadata_dir[0])
285		✗	spacemap_dir = s->metadata_dir;
286
287		✗	s->fd = s->mapfd = -1; /* Set these early for shared_close() failure path */
288
289			/* Open underlying protocol */
290		✗	av_strstart(arg, "shared:", &arg);
291		✗	ret = ffurl_open_whitelist(&s->inner, arg, flags, &h->interrupt_callback,
292			options, h->protocol_whitelist, h->protocol_blacklist, h);
293		✗	if (is_ignorable_error(ret) && s->ignore_errors) {
294		✗	av_log(h, AV_LOG_WARNING, "Underlying URL failed to open: %s. "
295		✗	"Continuing with cache file only.\n", av_err2str(ret));
296		✗	} else if (ret < 0)
297		✗	goto fail;
298
299			uint8_t hash[HASH_SIZE];
300		✗	ret = hash_uri(hash, arg);
301		✗	if (ret < 0)
302		✗	goto fail;
303
304			/* 128 bits is enough for collision resistance; we already store the full
305			* hash inside the header for verification */
306			char filename[2 * 16 + 1];
307		✗	ff_data_to_hex(filename, hash, sizeof(filename)/2, 0);
308		✗	s->cache_path = av_asprintf("%s/%s.cache", s->cache_dir, filename);
309		✗	s->map_path = av_asprintf("%s/%s.spacemap", spacemap_dir, filename);
310		✗	if (!s->cache_path || !s->map_path) {
311		✗	ret = AVERROR(ENOMEM);
312		✗	goto fail;
313			}
314
315		✗	av_log(h, AV_LOG_VERBOSE, "Opening cache file '%s' for URI: '%s'\n",
316		✗	s->cache_path, s->inner ? s->inner->filename : arg);
317
318		✗	const int mode = O_RDWR | O_BINARY | (s->inner ? O_CREAT : 0);
319		✗	s->fd = avpriv_open(s->cache_path, mode, 0660);
320		✗	s->mapfd = s->fd >= 0 ? avpriv_open(s->map_path, mode, 0660) : -1;
321		✗	if (s->fd < 0 || s->mapfd < 0) {
322		✗	ret = AVERROR(errno);
323		✗	av_log(h, AV_LOG_ERROR, "Failed to open '%s': %s\n",
324		✗	s->fd < 0 ? s->cache_path : s->map_path, av_err2str(ret));
325		✗	goto fail;
326			}
327
328		✗	ret = spacemap_init(h, hash);
329		✗	if (ret < 0)
330		✗	goto fail;
331
332			/* s->block_shift is fully settled after spacemap_init() */
333		✗	s->block_size = 1 << s->block_shift;
334		✗	s->blocks_max = s->cache_size_max >> s->block_shift;
335
336		✗	int64_t filesize = get_filesize(h);
337		✗	if (filesize < 0) {
338		✗	ret = (int) filesize;
339		✗	goto fail;
340		✗	} else if (!filesize) {
341			/* Filesize is not yet known, try to get it from the underlying URL;
342			* go through our own seek function to handle errors and updates */
343		✗	filesize = ffurl_size(h);
344		✗	if (filesize < 0 && filesize != AVERROR(ENOSYS)) {
345		✗	ret = (int) filesize;
346		✗	goto fail;
347			}
348			}
349
350		✗	if (filesize > 0) {
351		✗	int64_t last_pos = filesize - 1;
352		✗	int64_t last_block = last_pos >> s->block_shift;
353		✗	ret = spacemap_grow(h, last_block);
354		✗	if (ret < 0)
355		✗	goto fail;
356			}
357
358			/* Temporary buffer needed for pread/pwrite() fallback */
359		✗	s->tmp_buf = av_malloc(s->block_size);
360		✗	if (!s->tmp_buf) {
361		✗	ret = AVERROR(ENOMEM);
362		✗	goto fail;
363			}
364
365		✗	h->max_packet_size = s->block_size;
366		✗	h->min_packet_size = s->block_size;
367		✗	ret = 0;
368
369		✗	fail:
370		✗	if (ret < 0)
371		✗	shared_close(h);
372		✗	return ret;
373			}
374
375		✗	static int spacemap_remap(URLContext *h, size_t map_size)
376			{
377		✗	SharedContext *s = h->priv_data;
378		✗	int ret, did_grow = 0, locked = 0;
379		✗	if (map_size <= s->map_size)
380		✗	return 0;
381
382			/* Opportunistically get current filesize before attempting to lock */
383			struct stat st;
384		✗	ret = fstat(s->mapfd, &st);
385		✗	if (ret < 0) {
386		✗	ret = AVERROR(errno);
387		✗	goto fail;
388			}
389
390		✗	if (st.st_size >= map_size)
391		✗	goto skip_resize;
392
393			/* Lock the spacemap to ensure nobody else is currently resizing it */
394		✗	ret = flock(s->mapfd, LOCK_EX);
395		✗	if (ret < 0) {
396		✗	ret = AVERROR(errno);
397		✗	goto fail;
398			}
399		✗	locked = 1;
400
401			/* Refresh filesize after acquiring the lock */
402		✗	ret = fstat(s->mapfd, &st);
403		✗	if (ret < 0) {
404		✗	ret = AVERROR(errno);
405		✗	goto fail;
406			}
407
408		✗	if (st.st_size >= map_size)
409		✗	goto skip_resize;
410
411			/* The new mapping extends the file */
412		✗	st.st_size = map_size;
413		✗	did_grow = 1;
414
415		✗	skip_resize:
416		✗	av_file_unmap_shared(s->spacemap, s->map_size);
417		✗	s->spacemap = NULL;
418		✗	s->map_size = st.st_size;
419
420			void *map;
421		✗	ret = av_file_map_shared(s->mapfd, s->map_size, &map);
422		✗	if (ret < 0) {
423		✗	s->map_size = 0;
424		✗	goto fail;
425			}
426		✗	s->spacemap = map;
427
428		✗	if (locked) {
429		✗	flock(s->mapfd, LOCK_UN);
430		✗	locked = 0;
431			}
432
433		✗	return did_grow;
434
435		✗	fail:
436		✗	if (locked)
437		✗	flock(s->mapfd, LOCK_UN);
438		✗	av_log(h, AV_LOG_ERROR, "Failed to resize space map: %s\n", av_err2str(ret));
439		✗	return ret;
440			}
441
442		✗	static int spacemap_grow(URLContext *h, int64_t block)
443			{
444		✗	SharedContext *s = h->priv_data;
445		✗	int64_t num_blocks = block + 1;
446		✗	size_t map_bytes = sizeof(Spacemap) + num_blocks * sizeof(Block);
447
448			/* When streaming files without known size, round up the number of blocks
449			* to the nearest multiple of the block size to reduce the rate of resizes */
450		✗	int64_t filesize = get_filesize(h);
451		✗	if (filesize < 0)
452		✗	return (int) filesize;
453		✗	else if (!filesize) {
454		✗	av_assert0(s->block_size > 0);
455		✗	map_bytes = FFALIGN(map_bytes, (int64_t) s->block_size);
456			}
457
458		✗	if (map_bytes < num_blocks)
459		✗	return AVERROR(EINVAL); /* overflow */
460
461		✗	const off_t old_size = s->map_size;
462		✗	int ret = spacemap_remap(h, map_bytes);
463		✗	if (ret < 0)
464		✗	return ret;
465
466			/* Report new size after successful grow */
467		✗	if (s->map_size > old_size) {
468		✗	num_blocks = (s->map_size - sizeof(Spacemap)) / sizeof(Block);
469		✗	av_log(h, AV_LOG_DEBUG,
470			"%s %zu bytes, capacity: %"PRId64" blocks = %"PRId64" MB\n",
471			ret ? "Resized spacemap to" : "Mapped spacemap with",
472		✗	(size_t) s->map_size, num_blocks,
473		✗	(num_blocks * (int64_t) s->block_size) >> 20);
474			}
475		✗	return 0;
476			}
477
478		✗	static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE])
479			{
480		✗	SharedContext *s = h->priv_data;
481			int ret;
482
483		✗	ret = spacemap_remap(h, sizeof(Spacemap));
484		✗	if (ret < 0)
485		✗	return ret;
486
487		✗	if ((ret = set_once_uint(&s->spacemap->header_magic, HEADER_MAGIC)) < 0 ||
488		✗	(ret = set_once_ushort(&s->spacemap->version, HEADER_VERSION)) < 0)
489			{
490		✗	av_log(h, AV_LOG_ERROR, "Shared cache spacemap header mismatch!\n");
491		✗	av_log(h, AV_LOG_ERROR, " Expected magic: 0x%X, version: %d\n",
492			HEADER_MAGIC, HEADER_VERSION);
493		✗	av_log(h, AV_LOG_ERROR, " Got magic: 0x%X, version: %d\n",
494		✗	atomic_load(&s->spacemap->header_magic),
495		✗	atomic_load(&s->spacemap->version));
496		✗	return ret;
497			}
498
499		✗	ret = set_once_ushort(&s->spacemap->block_shift, s->block_shift);
500		✗	if (ret < 0) {
501		✗	const int shift = atomic_load(&s->spacemap->block_shift);
502		✗	av_log(h, AV_LOG_WARNING, "Shared cache uses block shift %d, "
503			"but requested block shift is %d.\n", shift, s->block_shift);
504		✗	if (shift < 9 || shift > 30) {
505		✗	av_log(h, AV_LOG_ERROR, "Invalid block shift %d in cache file!\n", shift);
506		✗	return AVERROR(EINVAL);
507			}
508		✗	s->block_shift = shift;
509			}
510
511		✗	for (int i = 0; i < HASH_SIZE; i++) {
512		✗	ret = set_once_uchar(&s->spacemap->hash[i], hash[i]);
513		✗	if (ret < 0) {
514		✗	av_log(h, AV_LOG_ERROR, "Shared cache spacemap hash mismatch!\n");
515			char hash_hex[2 * HASH_SIZE + 1];
516		✗	ff_data_to_hex(hash_hex, hash, HASH_SIZE, 0);
517		✗	av_log(h, AV_LOG_ERROR, " Expected hash: %s\n", hash_hex);
518			uint8_t hash2[HASH_SIZE];
519		✗	for (int j = 0; j < HASH_SIZE; ++j)
520		✗	hash2[j] = atomic_load_explicit(&s->spacemap->hash[j], memory_order_relaxed);
521		✗	ff_data_to_hex(hash_hex, hash2, HASH_SIZE, 0);
522		✗	av_log(h, AV_LOG_ERROR, " Got hash: %s\n", hash_hex);
523		✗	return ret;
524			}
525			}
526
527		✗	if (ret) /* set_once() return 1 if this is the first time setting the value */
528		✗	av_log(h, AV_LOG_DEBUG, "Initialized new cache spacemap.\n");
529
530		✗	return ret;
531			}
532
533		✗	static int read_cache(SharedContext *s, uint8_t *buf, size_t size, off_t offset)
534			{
535		✗	while (size) {
536		✗	ssize_t ret = pread(s->fd, buf, size, offset);
537		✗	if (ret <= 0)
538		✗	return ret ? AVERROR(errno) : AVERROR_EOF;
539		✗	buf += ret;
540		✗	offset += ret;
541		✗	size -= ret;
542			}
543
544		✗	return 0;
545			}
546
547		✗	static int write_cache(SharedContext *s, const uint8_t *buf, size_t size, off_t offset)
548			{
549		✗	while (size) {
550		✗	ssize_t ret = pwrite(s->fd, buf, size, offset);
551		✗	if (ret <= 0)
552		✗	return ret ? AVERROR(errno) : AVERROR(EIO);
553		✗	buf += ret;
554		✗	offset += ret;
555		✗	size -= ret;
556			}
557
558		✗	return 0;
559			}
560
561		✗	static int clamp_size(URLContext *h, int size, int64_t pos, int64_t filesize)
562			{
563		✗	if (!filesize)
564		✗	return size;
565		✗	else if (pos > filesize)
566		✗	return 0;
567			else
568		✗	return FFMIN(filesize - pos, size);
569			}
570
571		✗	static int shared_read(URLContext *h, unsigned char *buf, int size)
572			{
573		✗	SharedContext *s = h->priv_data;
574			uint8_t *tmp;
575			int ret;
576		✗	if (!s->spacemap)
577		✗	return AVERROR(EIO);
578
579		✗	if (size <= 0)
580		✗	return 0;
581
582		✗	int64_t filesize = get_filesize(h);
583		✗	if (filesize < 0)
584		✗	return (int) filesize;
585
586		✗	size = clamp_size(h, size, s->pos, filesize);
587		✗	if (size <= 0)
588		✗	return AVERROR_EOF;
589
590		✗	const int64_t block_id = s->pos >> s->block_shift;
591		✗	const int64_t offset = s->pos & (s->block_size - 1);
592		✗	const int64_t block_pos = block_id * s->block_size;
593		✗	int block_size = clamp_size(h, s->block_size, block_pos, filesize);
594		✗	ret = spacemap_grow(h, block_id);
595		✗	if (ret < 0)
596		✗	return ret;
597
598		✗	Block *const block = &s->spacemap->blocks[block_id];
599		✗	unsigned state = atomic_load_explicit(&block->state, memory_order_acquire);
600		✗	int64_t pending_since = 0;
601		✗	int verify_read = 0, acquired = 0, allocated = 0;
602
603		✗	retry:
604		✗	switch (state) {
605		✗	default:
606		✗	if (s->num_corrupt >= MAX_CORRUPT_BLOCKS)
607		✗	goto read_block; /* assume broken cache file */
608
609			/* filesize may have become known in the meantime */
610		✗	filesize = get_filesize(h);
611		✗	if (filesize < 0)
612		✗	return (int) filesize;
613
614			/* We always need to read the entire block to verify integrity */
615		✗	block_size = clamp_size(h, block_size, block_pos, filesize);
616		✗	tmp = s->tmp_buf;
617		✗	ret = read_cache(s, tmp, block_size, block_pos);
618		✗	if (ret < 0) {
619		✗	av_log(h, AV_LOG_ERROR, "Failed to read from cache file: %s\n", av_err2str(ret));
620		✗	if (ret == AVERROR_EOF) { /* e.g. cache appears truncated? */
621		✗	if (s->retry_corrupt) {
622		✗	s->num_corrupt++;
623		✗	goto read_block;
624			}
625		✗	ret = AVERROR(EIO); /* don't propagate EOF to caller */
626			}
627		✗	return ret;
628			}
629
630		✗	uint32_t crc = get_block_crc(tmp, block_size);
631		✗	if (crc != state) {
632		✗	av_log(h, AV_LOG_ERROR, "Cache corruption detected for block 0x%"PRIx64" at "
633			"offset 0x%"PRIx64": expected CRC: 0x%08X, got: 0x%08X\n",
634			block_id, block_pos, state, crc);
635		✗	if (s->retry_corrupt) {
636		✗	s->num_corrupt++;
637		✗	goto read_block;
638			}
639		✗	return AVERROR(EIO);
640			} else
641		✗	s->num_corrupt = 0; /* reset corrupt block count on success */
642
643		✗	tmp += (ptrdiff_t) offset;
644		✗	size = FFMIN(size, block_size - offset);
645		✗	if (size <= 0)
646		✗	return AVERROR_EOF;
647		✗	if (s->verify) {
648		✗	verify_read = 1;
649		✗	break; /* fall through to the cache miss logic */
650			}
651
652		✗	memcpy(buf, tmp, size);
653		✗	s->nb_hit++;
654		✗	s->pos += size;
655		✗	return size;
656
657		✗	case BLOCK_FAILED:
658		✗	if (s->retry_errors)
659		✗	goto read_block;
660		✗	return AVERROR(EIO);
661
662		✗	read_block:
663		✗	if (s->num_corrupt == MAX_CORRUPT_BLOCKS) {
664		✗	av_log(h, AV_LOG_ERROR, "Too many consecutive corrupt blocks; "
665			"assuming cache file is completely broken.\n");
666		✗	s->num_corrupt++; /* silence this log on subsequent reads */
667			}
668			av_fallthrough;
669
670			case BLOCK_NONE:
671		✗	if (s->read_only || s->write_err || !s->inner)
672			break; /* don't mark block as pending */
673		✗	else if (s->cache_size_max) {
674		✗	int64_t cached = atomic_load_explicit(&s->spacemap->blocks_cached,
675			memory_order_relaxed);
676		✗	if (cached >= s->blocks_max) {
677		✗	av_log(h, AV_LOG_WARNING, "Cache size limit reached (%"PRId64" "
678			"blocks = %"PRId64" bytes), switching to read-only mode.\n",
679		✗	s->blocks_max, s->blocks_max << s->block_shift);
680		✗	s->read_only = 1;
681		✗	break;
682			}
683			}
684
685		✗	if (atomic_compare_exchange_strong_explicit(&block->state, &state,
686			BLOCK_PENDING,
687			memory_order_acquire,
688			memory_order_acquire))
689			{
690			/* Acquired pending state, proceed to fetch the block */
691		✗	acquired = 1;
692		✗	allocated = (state == BLOCK_NONE || state == BLOCK_FAILED);
693		✗	state = BLOCK_PENDING;
694		✗	break;
695			}
696			/* CAS failed, another thread changed the state; reload it */
697		✗	goto retry;
698
699		✗	case BLOCK_PENDING:
700			/* Another thread is busy fetching this block, wait for it to finish */
701		✗	if (!s->timeout) {
702		✗	break; /* no timeout requested, immediately race to fetch block */
703		✗	} else if (pending_since) {
704		✗	int64_t new = av_gettime_relative();
705		✗	if (new - pending_since >= s->timeout)
706		✗	break; /* timeout expired, try to fetch the block ourselves */
707			} else {
708		✗	pending_since = av_gettime_relative();
709			}
710
711		✗	if (h->flags & AVIO_FLAG_NONBLOCK)
712		✗	return AVERROR(EAGAIN);
713
714			/* Make sure we try a few times before giving up */
715		✗	av_usleep(FFMIN(s->timeout >> 4, 10000));
716		✗	if (ff_check_interrupt(&h->interrupt_callback))
717		✗	return AVERROR_EXIT;
718
719		✗	state = atomic_load_explicit(&block->state, memory_order_acquire);
720		✗	goto retry;
721			}
722
723			/* Release pending state on failure to avoid stalling other threads */
724			#define RELEASE_PENDING(block, state) \
725			do { \
726			if (acquired) { \
727			av_assert1(state == BLOCK_PENDING); \
728			atomic_compare_exchange_strong_explicit( \
729			&block->state, &state, BLOCK_NONE, memory_order_relaxed, \
730			memory_order_relaxed); \
731			} \
732			} while (0)
733
734			/* Cache miss, fetch this block from underlying protocol */
735		✗	s->nb_miss++;
736
737		✗	if (!s->inner) {
738		✗	av_log(h, AV_LOG_ERROR, "Cache miss for block 0x%"PRIx64" at offset "
739			"0x%"PRIx64", but underlying protocol is not available!\n",
740			block_id, block_pos);
741		✗	av_assert0(!acquired);
742		✗	return AVERROR(EIO);
743			}
744
745		✗	const int read_only = s->read_only || s->write_err || verify_read;
746		✗	int64_t inner_pos = read_only ? s->pos : block_pos;
747		✗	if (s->inner_pos != inner_pos) {
748		✗	inner_pos = ffurl_seek(s->inner, inner_pos, SEEK_SET);
749		✗	if (inner_pos < 0) {
750		✗	av_log(h, AV_LOG_ERROR, "Failed to seek underlying protocol: %s\n",
751		✗	av_err2str(inner_pos));
752		✗	RELEASE_PENDING(block, state);
753		✗	return inner_pos;
754			}
755
756		✗	av_log(h, AV_LOG_DEBUG, "Inner seek to 0x%"PRIx64"\n", inner_pos);
757		✗	s->inner_pos = inner_pos;
758			}
759
760		✗	if (read_only) {
761			/* Directly defer to the underlying protocol */
762		✗	ret = ffurl_read(s->inner, buf, size);
763		✗	if (ret < 0) {
764			av_assert1(!acquired);
765		✗	return ret;
766			} else {
767		✗	s->inner_pos = inner_pos + ret;
768			}
769
770			/* Verify the read data against the cached data if requested */
771		✗	if (verify_read && memcmp(buf, tmp, ret)) {
772		✗	av_log(h, AV_LOG_ERROR, "Cache verification failed for %d bytes "
773			"in block 0x%"PRIx64" at offset 0x%"PRIx64" + %"PRId64"!\n",
774			ret, block_id, block_pos, offset);
775		✗	return AVERROR(EIO);
776			}
777
778		✗	s->pos = s->inner_pos;
779		✗	return ret;
780			}
781
782		✗	int write_back = 1;
783		✗	if (size >= block_size && !offset) {
784			/* Read directly into output buffer if aligned and large enough */
785		✗	tmp = buf;
786			} else {
787			/* Read into temporary buffer and copy later */
788		✗	tmp = s->tmp_buf;
789			}
790
791			/* Try and fetch the entire block */
792		✗	av_assert0(inner_pos == block_pos);
793		✗	int bytes_read = 0;
794		✗	while (bytes_read < block_size) {
795		✗	ret = ffurl_read(s->inner, &tmp[bytes_read], block_size - bytes_read);
796		✗	if (!ret || ret == AVERROR_EOF)
797			break;
798		✗	else if (ret < 0) {
799		✗	av_log(h, AV_LOG_ERROR, "Failed to read block 0x%"PRIx64": %s\n",
800		✗	block_id, av_err2str(ret));
801		✗	if (ret == AVERROR(EAGAIN) || ret == AVERROR_EXIT) {
802		✗	RELEASE_PENDING(block, state);
803		✗	return ret; /* transient error, allow retries */
804			}
805
806			/* Try to mark block as failed; ignore errors - any mismatch
807			* here will mean that either another thread already marked it
808			* as failed, or successfully cached it in the meantime */
809		✗	atomic_compare_exchange_strong_explicit(&block->state, &state,
810			BLOCK_FAILED,
811			memory_order_relaxed,
812			memory_order_relaxed);
813		✗	return ret;
814			}
815
816		✗	bytes_read += ret;
817		✗	s->inner_pos += ret;
818			}
819
820		✗	if (bytes_read < block_size) {
821			/* Learned location of true EOF, update filesize */
822		✗	ret = set_filesize(h, inner_pos + bytes_read);
823		✗	if (ret < 0) {
824		✗	RELEASE_PENDING(block, state);
825		✗	return ret;
826			}
827			}
828
829		✗	if (bytes_read > 0) {
830		✗	ret = write_back ? write_cache(s, tmp, bytes_read, block_pos) : 0;
831		✗	if (ret < 0) {
832		✗	if (ret != AVERROR(EINTR)) {
833		✗	av_log(h, AV_LOG_ERROR, "Failed to write to cache file: %s\n",
834		✗	av_err2str(ret));
835		✗	s->write_err = 1;
836			}
837		✗	RELEASE_PENDING(block, state);
838			} else {
839		✗	uint32_t crc = get_block_crc(tmp, bytes_read);
840		✗	av_log(h, AV_LOG_TRACE, "Cached %d bytes to block 0x%"PRIx64" at "
841			"offset 0x%"PRIx64", CRC 0x%08X\n", bytes_read, block_id,
842			block_pos, crc);
843		✗	atomic_store_explicit(&block->state, crc, memory_order_release);
844		✗	if (allocated)
845		✗	atomic_fetch_add_explicit(&s->spacemap->blocks_cached, 1, memory_order_release);
846			}
847			} else {
848		✗	RELEASE_PENDING(block, state);
849		✗	return AVERROR_EOF;
850			}
851
852		✗	size = FFMIN(bytes_read - offset, size);
853		✗	if (size <= 0)
854		✗	return AVERROR_EOF;
855		✗	if (tmp != buf)
856		✗	memcpy(buf, &tmp[offset], size);
857		✗	s->pos += size;
858		✗	return size;
859			}
860
861		✗	static int64_t shared_seek(URLContext *h, int64_t pos, int whence)
862			{
863		✗	SharedContext *s = h->priv_data;
864			int64_t res;
865		✗	if (!s->spacemap)
866		✗	return AVERROR(EIO);
867
868		✗	const int64_t filesize = get_filesize(h);
869		✗	if (filesize < 0)
870		✗	return filesize;
871
872		✗	switch (whence) {
873		✗	case AVSEEK_SIZE:
874		✗	if (filesize)
875		✗	return filesize;
876		✗	res = s->inner ? ffurl_seek(s->inner, pos, whence) : AVERROR(ENOSYS);
877		✗	if (res > 0) {
878		✗	if (set_filesize(h, res) < 0)
879		✗	return AVERROR(EINVAL);
880		✗	} else if (is_ignorable_error(res) && s->ignore_errors) {
881		✗	av_log(h, AV_LOG_WARNING, "Underlying URL failed to get size: %s. "
882		✗	"Continuing with cache file only.\n", av_err2str(res));
883		✗	ffurl_closep(&s->inner);
884		✗	res = AVERROR(ENOSYS);
885			}
886		✗	return res;
887		✗	case SEEK_SET:
888		✗	break;
889		✗	case SEEK_CUR:
890		✗	pos += s->pos;
891		✗	break;
892		✗	case SEEK_END:
893		✗	if (filesize) {
894		✗	pos += filesize;
895		✗	break;
896			}
897
898			/* Defer to underlying protocol if filesize is unknown */
899		✗	res = s->inner ? ffurl_seek(s->inner, pos, whence) : AVERROR(ENOSYS);
900		✗	if (is_ignorable_error(res) && s->ignore_errors) {
901		✗	av_log(h, AV_LOG_WARNING, "Underlying URL failed to seek: %s. "
902		✗	"Continuing with cache file only.\n", av_err2str(res));
903		✗	ffurl_closep(&s->inner);
904		✗	return AVERROR(ENOSYS);
905		✗	} else if (res < 0)
906		✗	return res;
907
908			/* Opportunistically update known filesize */
909		✗	if (set_filesize(h, res - pos) < 0)
910		✗	return AVERROR(EINVAL);
911		✗	av_log(h, AV_LOG_DEBUG, "Inner seek to 0x%"PRIx64"\n", res);
912		✗	return s->pos = s->inner_pos = res;
913		✗	default:
914		✗	return AVERROR(EINVAL);
915			}
916
917		✗	if (pos < 0)
918		✗	return AVERROR(EINVAL);
919
920		✗	av_log(h, AV_LOG_DEBUG, "Virtual seek to 0x%"PRIx64"\n", pos);
921		✗	return s->pos = pos;
922			}
923
924		✗	static int shared_get_file_handle(URLContext *h)
925			{
926		✗	SharedContext *s = h->priv_data;
927		✗	return s->inner ? ffurl_get_file_handle(s->inner) : -1;
928			}
929
930		✗	static int shared_get_short_seek(URLContext *h)
931			{
932		✗	SharedContext *s = h->priv_data;
933		✗	int ret = s->inner ? ffurl_get_short_seek(s->inner) : 0;
934		✗	return ret > 0 ? FFMAX(ret, s->block_size) : s->block_size;
935			}
936
937			#define OFFSET(x) offsetof(SharedContext, x)
938			#define D AV_OPT_FLAG_DECODING_PARAM
939
940			static const AVOption options[] = {
941			{ "cache_dir", "Directory path for shared file cache", OFFSET(cache_dir), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D },
942			{ "metadata_dir", "Directory path for shared file cache metadata", OFFSET(metadata_dir), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D },
943			{ "block_shift", "Set the base 2 logarithm of the block size", OFFSET(block_shift), AV_OPT_TYPE_INT, {.i64 = 15}, 9, 30, .flags = D },
944			{ "read_only", "Don't write data to the cache, only read from it", OFFSET(read_only), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
945			{ "cache_verify", "Verify correctness of the cache against the source", OFFSET(verify), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
946			{ "cache_timeout", "Time in us to wait before re-fetching pending blocks", OFFSET(timeout), AV_OPT_TYPE_INT64, {.i64 = 10000}, 0, INT64_MAX, .flags = D },
947			{ "ignore_errors", "Continue even if the inner URL failed", OFFSET(ignore_errors), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
948			{ "retry_errors", "Re-request blocks even if they previously failed", OFFSET(retry_errors), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
949			{ "retry_corrupt", "Re-request blocks that fail the CRC check", OFFSET(retry_corrupt), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
950			{ "cache_size_max", "Limit the maximum amount of data cached", OFFSET(cache_size_max), AV_OPT_TYPE_INT64, {.i64 = 0}, 0, INT64_MAX, .flags = D },
951			{0},
952			};
953
954			static const AVClass shared_context_class = {
955			.class_name = "shared",
956			.item_name = av_default_item_name,
957			.option = options,
958			.version = LIBAVUTIL_VERSION_INT,
959			};
960
961			const URLProtocol ff_shared_protocol = {
962			.name = "shared",
963			.url_open2 = shared_open,
964			.url_read = shared_read,
965			.url_seek = shared_seek,
966			.url_close = shared_close,
967			.url_get_file_handle = shared_get_file_handle,
968			.url_get_short_seek = shared_get_short_seek,
969			.priv_data_size = sizeof(SharedContext),
970			.priv_data_class = &shared_context_class,
971			};
972