FFmpeg coverage

Line	Branch	Exec	Source
1			/*
2			* Shared file cache protocol.
3			* Copyright (c) 2026 Niklas Haas
4			*
5			* This file is part of FFmpeg.
6			*
7			* FFmpeg is free software; you can redistribute it and/or
8			* modify it under the terms of the GNU Lesser General Public
9			* License as published by the Free Software Foundation; either
10			* version 2.1 of the License, or (at your option) any later version.
11			*
12			* FFmpeg is distributed in the hope that it will be useful,
13			* but WITHOUT ANY WARRANTY; without even the implied warranty of
14			* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15			* Lesser General Public License for more details.
16			*
17			* You should have received a copy of the GNU Lesser General Public
18			* License along with FFmpeg; if not, write to the Free Software
19			* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20			*
21			* Based on cache.c by Michael Niedermayer
22			*/
23
24			#include "libavutil/attributes.h"
25			#include "libavutil/avassert.h"
26			#include "libavutil/avstring.h"
27			#include "libavutil/crc.h"
28			#include "libavutil/error.h"
29			#include "libavutil/hash.h"
30			#include "libavutil/file_open.h"
31			#include "libavutil/mem.h"
32			#include "libavutil/opt.h"
33			#include "libavutil/time.h"
34
35			#include "url.h"
36
37			#include <errno.h>
38			#include <fcntl.h>
39			#include <inttypes.h>
40			#include <stdatomic.h>
41			#include <string.h>
42			#include <sys/file.h>
43			#include <sys/mman.h>
44			#include <sys/stat.h>
45			#include <unistd.h>
46
47			/**
48			* This hash should be resistant against collision attacks, so that an
49			* attacker could not generate e.g. two different URIs that map to the same
50			* cache file. This requires at least 64 bits of collision resistance in
51			* practice (i.e. 128 bits = 16 bytes of hash size). However, we can be
52			* conservative by computing e.g. a 256 bit hash and storing it inside the
53			* file header for verification.
54			*
55			* Note that due to the way we use atomics, we should avoid zero bytes in
56			* the resulting hash; hence we tweak the input slightly to avoid this.
57			* The resulting loss in hash strength is negligible, since 32 bytes is
58			* already much more than needed.
59			*/
60			#define HASH_METHOD "SHA512/256"
61			#define HASH_SIZE 32
62			#define HEADER_MAGIC MKTAG(u'\xFF', 'S', 'h', '$')
63			#define HEADER_VERSION 3
64
65			/**
66			* Hard watershed of consecutive failed blocks before we give up on the cache
67			* file altogether and assume it's entirely lost to us.
68			**/
69			#define MAX_CORRUPT_BLOCKS 10
70
71		✗	static int hash_uri(uint8_t hash[HASH_SIZE], const char *uri)
72			{
73		✗	struct AVHashContext *ctx = NULL;
74		✗	int ret = av_hash_alloc(&ctx, HASH_METHOD);
75		✗	if (ret < 0)
76		✗	return ret;
77
78		✗	const int16_t version = HEADER_VERSION;
79		✗	av_assert0(av_hash_get_size(ctx) == HASH_SIZE);
80		✗	av_hash_init(ctx);
81		✗	av_hash_update(ctx, (const uint8_t *) &version, sizeof(version));
82		✗	av_hash_update(ctx, (const uint8_t *) uri, strlen(uri));
83		✗	av_hash_final(ctx, hash);
84		✗	av_hash_freep(&ctx);
85
86		✗	for (int i = 0; i < HASH_SIZE; i++)
87		✗	hash[i] = hash[i] ? hash[i] : ~hash[i]; /* prevent zero bytes */
88		✗	return 0;
89			}
90
91			enum BlockState {
92			/* Reserved block state values */
93			BLOCK_NONE = 0, ///< block is not cached
94			BLOCK_PENDING, ///< a thread is currently trying to write this block
95			BLOCK_FAILED, ///< the underlying I/O source failed to read this block
96
97			/**
98			* All other block states represent valid cached blocks, with the value
99			* being the CRC of the block data.
100			*/
101			};
102
103		✗	static uint32_t get_block_crc(const uint8_t *block, size_t block_size)
104			{
105		✗	uint32_t crc = av_crc(av_crc_get_table(AV_CRC_32_IEEE), 0, block, block_size);
106		✗	switch (crc) {
107		✗	case BLOCK_NONE:
108			case BLOCK_FAILED:
109			case BLOCK_PENDING:
110		✗	return ~crc; /* avoid reserved block states */
111		✗	default:
112		✗	return crc;
113			}
114			}
115
116			typedef struct Block {
117			atomic_uint state; /* enum BlockState */
118			} Block;
119
120			typedef struct Spacemap {
121			atomic_uint header_magic;
122			atomic_ushort version;
123			atomic_ushort block_shift;
124			atomic_ullong filesize; /* byte offset of true EOF, or 0 if unknown */
125			atomic_uchar hash[HASH_SIZE]; /* hash of resource URI / filename */
126			char reserved[80];
127
128			Block blocks[];
129			} Spacemap;
130
131			/* Set to value iff the current value is unset (zero) */
132			#define DEF_SET_ONCE(ctype, atype) \
133			static int set_once_##atype(atomic_##atype *const ptr, const ctype value) \
134			{ \
135			ctype prev = 0; \
136			av_assert1(value != 0); \
137			if (atomic_compare_exchange_strong_explicit( \
138			ptr, &prev, value, memory_order_release, memory_order_relaxed)) \
139			return 1; \
140			else if (prev == value) \
141			return 0; \
142			else \
143			return AVERROR(EINVAL); \
144			}
145
146		✗	DEF_SET_ONCE(unsigned char, uchar)
147		✗	DEF_SET_ONCE(unsigned int, uint)
148		✗	DEF_SET_ONCE(unsigned short, ushort)
149		✗	DEF_SET_ONCE(unsigned long long, ullong)
150
151			typedef struct SharedContext {
152			AVClass *class;
153			URLContext *inner;
154			int64_t inner_pos;
155
156			/* options */
157			char *cache_dir;
158			int block_shift; ///< requested shift; may disagree with actual
159			int read_only;
160			int64_t timeout;
161			int retry_errors;
162			int retry_corrupt;
163			int verify;
164
165			/* misc state */
166			int64_t pos; ///< current logical position
167			uint8_t *tmp_buf;
168			int block_size;
169			int write_err; ///< write error occurred
170			int num_corrupt;
171
172			/* cache file */
173			uint8_t *cache_data; ///< optional mmap of the cache file
174			char *cache_path;
175			off_t cache_size; ///< size of mapped memory region (for munmap)
176			int fd;
177
178			/* space map */
179			Spacemap *spacemap;
180			char *map_path;
181			off_t map_size;
182			int mapfd;
183
184			/* statistics */
185			int64_t nb_hit;
186			int64_t nb_miss;
187			} SharedContext;
188
189		✗	static int shared_close(URLContext *h)
190			{
191		✗	SharedContext *s = h->priv_data;
192
193		✗	ffurl_close(s->inner);
194		✗	if (s->cache_data)
195		✗	munmap(s->cache_data, s->cache_size);
196		✗	if (s->spacemap)
197		✗	munmap(s->spacemap, s->map_size);
198		✗	if (s->fd != -1)
199		✗	close(s->fd);
200		✗	if (s->mapfd != -1)
201		✗	close(s->mapfd);
202		✗	av_freep(&s->cache_path);
203		✗	av_freep(&s->map_path);
204		✗	av_freep(&s->tmp_buf);
205
206		✗	av_log(h, AV_LOG_DEBUG, "Cache statistics: %"PRId64" hits, %"PRId64" misses\n",
207			s->nb_hit, s->nb_miss);
208		✗	return 0;
209			}
210
211			static int cache_map(URLContext *h, int64_t filesize);
212			static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE]);
213			static int spacemap_grow(URLContext *h, int64_t block);
214
215		✗	static int64_t get_filesize(URLContext *h)
216			{
217		✗	SharedContext *s = h->priv_data;
218		✗	return atomic_load_explicit(&s->spacemap->filesize, memory_order_relaxed);
219			}
220
221		✗	static int set_filesize(URLContext *h, int64_t new_size)
222			{
223		✗	SharedContext *s = h->priv_data;
224			int ret;
225
226		✗	if (!new_size)
227		✗	return 0;
228
229		✗	ret = set_once_ullong(&s->spacemap->filesize, new_size);
230		✗	if (ret < 0) {
231		✗	av_log(h, AV_LOG_ERROR, "Cached file size mismatch, expected: "
232			"%"PRId64", got: %"PRIu64"!\n", new_size,
233		✗	(uint64_t) atomic_load(&s->spacemap->filesize));
234		✗	return ret;
235		✗	} else if (ret) {
236			/* Opportunistically map the file; this also sets the correct filesize.
237			* Ignore errors as this is not critical to the cache logic. */
238		✗	cache_map(h, new_size);
239			}
240
241		✗	return ret;
242			}
243
244		✗	static int shared_open(URLContext *h, const char *arg, int flags, AVDictionary **options)
245			{
246		✗	SharedContext *s = h->priv_data;
247			int ret;
248
249		✗	if (!s->cache_dir || !s->cache_dir[0]) {
250		✗	av_log(h, AV_LOG_ERROR, "Missing path for shared cache! Specify a "
251			"directory using the -cache_dir option.\n");
252		✗	return AVERROR(EINVAL);
253			}
254
255		✗	s->fd = s->mapfd = -1; /* Set these early for shared_close() failure path */
256
257			/* Open underlying protocol */
258		✗	av_strstart(arg, "shared:", &arg);
259		✗	ret = ffurl_open_whitelist(&s->inner, arg, flags, &h->interrupt_callback,
260			options, h->protocol_whitelist, h->protocol_blacklist, h);
261
262		✗	if (ret < 0)
263		✗	goto fail;
264
265			uint8_t hash[HASH_SIZE];
266		✗	ret = hash_uri(hash, arg);
267		✗	if (ret < 0)
268		✗	goto fail;
269
270			/* 128 bits is enough for collision resistance; we already store the full
271			* hash inside the header for verification */
272			char filename[2 * 16 + 1];
273		✗	for (int i = 0; i < FF_ARRAY_ELEMS(filename) / 2; i++)
274		✗	sprintf(&filename[i * 2], "%02X", hash[i]);
275		✗	s->cache_path = av_asprintf("%s/%s.cache", s->cache_dir, filename);
276		✗	s->map_path = av_asprintf("%s/%s.spacemap", s->cache_dir, filename);
277		✗	if (!s->cache_path || !s->map_path) {
278		✗	ret = AVERROR(ENOMEM);
279		✗	goto fail;
280			}
281
282		✗	av_log(h, AV_LOG_VERBOSE, "Opening cache file '%s' for URI: '%s'\n",
283		✗	s->cache_path, s->inner->filename);
284
285		✗	s->fd = avpriv_open(s->cache_path, O_RDWR | O_CREAT, 0660);
286		✗	s->mapfd = avpriv_open(s->map_path, O_RDWR | O_CREAT, 0660);
287		✗	if (s->fd < 0 || s->mapfd < 0) {
288		✗	ret = AVERROR(errno);
289		✗	av_log(h, AV_LOG_ERROR, "Failed to open '%s': %s\n",
290		✗	s->fd < 0 ? s->cache_path : s->map_path, av_err2str(ret));
291		✗	goto fail;
292			}
293
294		✗	ret = spacemap_init(h, hash);
295		✗	if (ret < 0)
296		✗	goto fail;
297
298		✗	s->block_size = 1 << atomic_load(&s->spacemap->block_shift);
299
300		✗	int64_t filesize = get_filesize(h);
301		✗	if (!filesize) {
302			/* Filesize is not yet known, try to get it from the underlying URL */
303		✗	filesize = ffurl_size(s->inner);
304		✗	if (filesize < 0 && filesize != AVERROR(ENOSYS)) {
305		✗	ret = (int) filesize;
306		✗	goto fail;
307		✗	} else if (filesize > 0) {
308		✗	ret = set_filesize(h, filesize);
309		✗	if (ret < 0)
310		✗	goto fail;
311			}
312			}
313
314		✗	if (filesize > 0) {
315		✗	int64_t last_pos = filesize - 1;
316		✗	int64_t last_block = last_pos >> atomic_load(&s->spacemap->block_shift);
317		✗	ret = spacemap_grow(h, last_block);
318		✗	if (ret < 0)
319		✗	goto fail;
320
321			/* If filesize is known, we can directly mmap() the cache file */
322		✗	ret = cache_map(h, filesize);
323		✗	if (ret < 0) {
324		✗	av_log(h, AV_LOG_WARNING, "Failed to map cache file: %s. Falling "
325		✗	"back to normal read/write\n", av_err2str(ret));
326		✗	ret = 0;
327			}
328			}
329
330			/* Temporary buffer needed for pread/pwrite() fallback */
331		✗	s->tmp_buf = av_malloc(s->block_size);
332		✗	if (!s->tmp_buf) {
333		✗	ret = AVERROR(ENOMEM);
334		✗	goto fail;
335			}
336
337		✗	h->max_packet_size = s->block_size;
338		✗	h->min_packet_size = s->block_size;
339		✗	ret = 0;
340
341		✗	fail:
342		✗	if (ret < 0)
343		✗	shared_close(h);
344		✗	return ret;
345			}
346
347		✗	static int cache_map(URLContext *h, int64_t filesize)
348			{
349		✗	SharedContext *s = h->priv_data;
350		✗	if (s->cache_size >= filesize || filesize > SIZE_MAX)
351		✗	return 0;
352
353		✗	if (s->cache_data) {
354		✗	munmap(s->cache_data, s->cache_size);
355		✗	s->cache_data = NULL;
356		✗	s->cache_size = 0;
357			}
358
359			struct stat st;
360		✗	int ret = fstat(s->fd, &st);
361		✗	if (ret < 0)
362		✗	return AVERROR(errno);
363
364		✗	if (st.st_size != filesize) {
365			/* Ensure the file size is correct before mapping; this can happen if
366			* another process wrote the correct filesize to the header but
367			* crashed right before actually successfully resizing the file. */
368		✗	ret = ftruncate(s->fd, filesize);
369		✗	if (ret < 0)
370		✗	return AVERROR(errno);
371			}
372
373		✗	s->cache_data = mmap(NULL, filesize, PROT_READ | PROT_WRITE, MAP_SHARED, s->fd, 0);
374		✗	if (s->cache_data == MAP_FAILED) {
375		✗	s->cache_data = NULL;
376		✗	return AVERROR(errno);
377			}
378
379		✗	s->cache_size = filesize;
380		✗	return 0;
381			}
382
383		✗	static int spacemap_remap(URLContext *h, size_t map_size)
384			{
385		✗	SharedContext *s = h->priv_data;
386		✗	int ret, did_grow = 0, locked = 0;
387		✗	if (map_size <= s->map_size)
388		✗	return 0;
389
390			/* Opportunistically get current filesize before attempting to lock */
391			struct stat st;
392		✗	ret = fstat(s->mapfd, &st);
393		✗	if (ret < 0) {
394		✗	ret = AVERROR(errno);
395		✗	goto fail;
396			}
397
398		✗	if (st.st_size >= map_size)
399		✗	goto skip_resize;
400
401			/* Lock the spacemap to ensure nobody else is currently resizing it */
402		✗	ret = flock(s->mapfd, LOCK_EX);
403		✗	if (ret < 0) {
404		✗	ret = AVERROR(errno);
405		✗	goto fail;
406			}
407		✗	locked = 1;
408
409			/* Refresh filesize after acquiring the lock */
410		✗	ret = fstat(s->mapfd, &st);
411		✗	if (ret < 0) {
412		✗	ret = AVERROR(errno);
413		✗	goto fail;
414			}
415
416		✗	if (st.st_size >= map_size)
417		✗	goto skip_resize;
418
419		✗	ret = ftruncate(s->mapfd, map_size);
420		✗	if (ret < 0) {
421		✗	ret = AVERROR(errno);
422		✗	goto fail;
423			}
424		✗	st.st_size = map_size;
425		✗	did_grow = 1;
426
427		✗	skip_resize:
428		✗	if (s->spacemap)
429		✗	munmap(s->spacemap, s->map_size);
430		✗	s->map_size = st.st_size;
431		✗	s->spacemap = mmap(NULL, s->map_size, PROT_READ | PROT_WRITE, MAP_SHARED, s->mapfd, 0);
432		✗	if (s->spacemap == MAP_FAILED) {
433		✗	s->spacemap = NULL; /* for munmap check */
434		✗	s->map_size = 0;
435		✗	ret = AVERROR(errno);
436		✗	goto fail;
437			}
438
439		✗	if (locked) {
440		✗	flock(s->mapfd, LOCK_UN);
441		✗	locked = 0;
442			}
443
444		✗	return did_grow;
445
446		✗	fail:
447		✗	if (locked)
448		✗	flock(s->mapfd, LOCK_UN);
449		✗	av_log(h, AV_LOG_ERROR, "Failed to resize space map: %s\n", av_err2str(ret));
450		✗	return ret;
451			}
452
453		✗	static int spacemap_grow(URLContext *h, int64_t block)
454			{
455		✗	SharedContext *s = h->priv_data;
456		✗	int64_t num_blocks = block + 1;
457		✗	size_t map_bytes = sizeof(Spacemap) + num_blocks * sizeof(Block);
458
459			/* When streaming files without known size, round up the number of blocks
460			* to the nearest multiple of the block size to reduce the rate of resizes */
461		✗	if (!get_filesize(h)) {
462		✗	av_assert0(s->block_size > 0);
463		✗	map_bytes = FFALIGN(map_bytes, (int64_t) s->block_size);
464			}
465
466		✗	if (map_bytes < num_blocks)
467		✗	return AVERROR(EINVAL); /* overflow */
468
469		✗	const off_t old_size = s->map_size;
470		✗	int ret = spacemap_remap(h, map_bytes);
471		✗	if (ret < 0)
472		✗	return ret;
473
474			/* Report new size after successful grow */
475		✗	if (s->map_size > old_size) {
476		✗	num_blocks = (s->map_size - sizeof(Spacemap)) / sizeof(Block);
477		✗	av_log(h, AV_LOG_DEBUG,
478			"%s %zu bytes, capacity: %"PRId64" blocks = %zu MB\n",
479			ret ? "Resized spacemap to" : "Mapped spacemap with",
480		✗	(size_t) s->map_size, num_blocks,
481		✗	(num_blocks * (int64_t) s->block_size) >> 20);
482			}
483		✗	return 0;
484			}
485
486		✗	static int spacemap_init(URLContext *h, const uint8_t hash[HASH_SIZE])
487			{
488		✗	SharedContext *s = h->priv_data;
489			int ret;
490
491		✗	ret = spacemap_remap(h, sizeof(Spacemap));
492		✗	if (ret < 0)
493		✗	return ret;
494
495		✗	if ((ret = set_once_uint(&s->spacemap->header_magic, HEADER_MAGIC)) < 0 ||
496		✗	(ret = set_once_ushort(&s->spacemap->version, HEADER_VERSION)) < 0)
497			{
498		✗	av_log(h, AV_LOG_ERROR, "Shared cache spacemap header mismatch!\n");
499		✗	av_log(h, AV_LOG_ERROR, " Expected magic: 0x%X, version: %d\n",
500			HEADER_MAGIC, HEADER_VERSION);
501		✗	av_log(h, AV_LOG_ERROR, " Got magic: 0x%X, version: %d\n",
502		✗	atomic_load(&s->spacemap->header_magic),
503		✗	atomic_load(&s->spacemap->version));
504		✗	return ret;
505			}
506
507		✗	ret = set_once_ushort(&s->spacemap->block_shift, s->block_shift);
508		✗	if (ret < 0) {
509		✗	const int shift = atomic_load(&s->spacemap->block_shift);
510		✗	av_log(h, AV_LOG_WARNING, "Shared cache uses block shift %d, "
511			"but requested block shift is %d.\n", shift, s->block_shift);
512		✗	if (shift < 9 || shift > 30) {
513		✗	av_log(h, AV_LOG_ERROR, "Invalid block shift %d in cache file!\n", shift);
514		✗	return AVERROR(EINVAL);
515			}
516			}
517
518		✗	for (int i = 0; i < HASH_SIZE; i++) {
519		✗	ret = set_once_uchar(&s->spacemap->hash[i], hash[i]);
520		✗	if (ret < 0) {
521		✗	av_log(h, AV_LOG_ERROR, "Shared cache spacemap hash mismatch!\n");
522		✗	av_log(h, AV_LOG_ERROR, " Expected hash: ");
523		✗	for (int j = 0; j < 32; j++)
524		✗	av_log(h, AV_LOG_ERROR, "%02X", hash[j]);
525		✗	av_log(h, AV_LOG_ERROR, "\n Got hash: ");
526		✗	for (int j = 0; j < 32; j++)
527		✗	av_log(h, AV_LOG_ERROR, "%02X", atomic_load(&s->spacemap->hash[j]));
528		✗	av_log(h, AV_LOG_ERROR, "\n");
529		✗	return ret;
530			}
531			}
532
533		✗	if (ret) /* set_once() return 1 if this is the first time setting the value */
534		✗	av_log(h, AV_LOG_DEBUG, "Initialized new cache spacemap.\n");
535
536		✗	return ret;
537			}
538
539		✗	static int read_cache(SharedContext *s, uint8_t *buf, size_t size, off_t offset)
540			{
541		✗	if (s->cache_data) {
542			av_assert1(offset + size <= s->cache_size);
543		✗	memcpy(buf, s->cache_data + offset, size);
544		✗	return 0;
545			}
546
547		✗	while (size) {
548		✗	ssize_t ret = pread(s->fd, buf, size, offset);
549		✗	if (ret <= 0)
550		✗	return ret ? AVERROR(errno) : AVERROR_EOF;
551		✗	buf += ret;
552		✗	offset += ret;
553		✗	size -= ret;
554			}
555
556		✗	return 0;
557			}
558
559		✗	static int write_cache(SharedContext *s, const uint8_t *buf, size_t size, off_t offset)
560			{
561		✗	if (s->cache_data) {
562			av_assert1(offset + size <= s->cache_size);
563		✗	memcpy(s->cache_data + offset, buf, size);
564		✗	return 0;
565			}
566
567		✗	while (size) {
568		✗	ssize_t ret = pwrite(s->fd, buf, size, offset);
569		✗	if (ret <= 0)
570		✗	return ret ? AVERROR(errno) : AVERROR(EIO);
571		✗	buf += ret;
572		✗	offset += ret;
573		✗	size -= ret;
574			}
575
576		✗	return 0;
577			}
578
579		✗	static size_t clamp_size(URLContext *h, size_t size, int64_t pos)
580			{
581		✗	const int64_t filesize = get_filesize(h);
582		✗	if (!filesize)
583		✗	return size;
584		✗	else if (pos > filesize)
585		✗	return 0;
586			else
587		✗	return FFMIN(filesize - pos, size);
588			}
589
590		✗	static int shared_read(URLContext *h, unsigned char *buf, int size)
591			{
592		✗	SharedContext *s = h->priv_data;
593			uint8_t *tmp;
594			int ret;
595
596		✗	if (size <= 0)
597		✗	return 0;
598
599		✗	size = clamp_size(h, size, s->pos);
600		✗	if (size <= 0)
601		✗	return AVERROR_EOF;
602
603		✗	const int shift = atomic_load_explicit(&s->spacemap->block_shift, memory_order_relaxed);
604		✗	const int64_t block_id = s->pos >> shift;
605		✗	const int64_t offset = s->pos & (s->block_size - 1);
606		✗	const int64_t block_pos = block_id * s->block_size;
607		✗	int block_size = clamp_size(h, s->block_size, block_pos);
608		✗	ret = spacemap_grow(h, block_id);
609		✗	if (ret < 0)
610		✗	return ret;
611
612		✗	Block *const block = &s->spacemap->blocks[block_id];
613		✗	unsigned state = atomic_load_explicit(&block->state, memory_order_acquire);
614		✗	int64_t pending_since = 0;
615		✗	int verify_read = 0, is_race = 0;
616
617		✗	retry:
618		✗	switch (state) {
619		✗	default:
620		✗	if (s->num_corrupt >= MAX_CORRUPT_BLOCKS)
621		✗	goto read_block; /* assume broken cache file */
622
623			/* We always need to read the entire block to verify integrity */
624		✗	block_size = clamp_size(h, block_size, block_pos); /* filesize may have changed */
625		✗	if (s->cache_data) {
626			av_assert1(block_pos + block_size <= s->cache_size);
627		✗	tmp = s->cache_data + block_pos;
628			} else {
629		✗	tmp = s->tmp_buf;
630		✗	ret = read_cache(s, tmp, block_size, block_pos);
631		✗	if (ret < 0) {
632		✗	av_log(h, AV_LOG_ERROR, "Failed to read from cache file: %s\n", av_err2str(ret));
633		✗	if (ret == AVERROR_EOF) { /* e.g. cache appears truncated? */
634		✗	if (s->retry_corrupt) {
635		✗	s->num_corrupt++;
636		✗	goto read_block;
637			}
638		✗	ret = AVERROR(EIO); /* don't propagate EOF to caller */
639			}
640		✗	return ret;
641			}
642			}
643
644		✗	uint32_t crc = get_block_crc(tmp, block_size);
645		✗	if (crc != state) {
646		✗	av_log(h, AV_LOG_ERROR, "Cache corruption detected for block 0x%"PRIx64" at "
647			"offset 0x%"PRIx64": expected CRC: 0x%08X, got: 0x%08X\n",
648			block_id, block_pos, state, crc);
649		✗	if (s->retry_corrupt) {
650		✗	s->num_corrupt++;
651		✗	goto read_block;
652			}
653		✗	return AVERROR(EIO);
654			} else
655		✗	s->num_corrupt = 0; /* reset corrupt block count on success */
656
657		✗	tmp += (ptrdiff_t) offset;
658		✗	size = FFMIN(size, block_size - offset);
659		✗	if (s->verify) {
660		✗	verify_read = 1;
661		✗	break; /* fall through to the cache miss logic */
662			}
663
664		✗	memcpy(buf, tmp, size);
665		✗	s->nb_hit++;
666		✗	s->pos += size;
667		✗	return size;
668
669		✗	case BLOCK_FAILED:
670		✗	if (s->retry_errors)
671		✗	goto read_block;
672		✗	return AVERROR(EIO);
673
674		✗	read_block:
675		✗	if (s->num_corrupt == MAX_CORRUPT_BLOCKS) {
676		✗	av_log(h, AV_LOG_ERROR, "Too many consecutive corrupt blocks; "
677			"assuming cache file is completely broken.\n");
678		✗	s->num_corrupt++; /* silence this log on subsequent reads */
679			}
680			av_fallthrough;
681
682			case BLOCK_NONE:
683		✗	if (s->read_only)
684		✗	break; /* don't mark block as pending */
685		✗	if (atomic_compare_exchange_strong_explicit(&block->state, &state,
686			BLOCK_PENDING,
687			memory_order_acquire,
688			memory_order_acquire))
689			{
690			/* Acquired pending state, proceed to fetch the block */
691		✗	state = BLOCK_PENDING;
692		✗	break;
693			}
694			/* CAS failed, another thread changed the state; reload it */
695		✗	goto retry;
696
697		✗	case BLOCK_PENDING:
698			/* Another thread is busy fetching this block, wait for it to finish */
699		✗	if (!s->timeout) {
700		✗	is_race = 1;
701		✗	break; /* no timeout requested, immediately race to fetch block */
702		✗	} else if (pending_since) {
703		✗	int64_t new = av_gettime_relative();
704		✗	if (new - pending_since >= s->timeout) {
705		✗	is_race = 1;
706		✗	break; /* timeout expired, try to fetch the block ourselves */
707			}
708			} else {
709		✗	pending_since = av_gettime_relative();
710			}
711
712			/* Make sure we try a few times before giving up */
713		✗	av_usleep(s->timeout >> 4);
714		✗	state = atomic_load_explicit(&block->state, memory_order_acquire);
715		✗	goto retry;
716			}
717
718			/* Cache miss, fetch this block from underlying protocol */
719		✗	s->nb_miss++;
720
721		✗	const int read_only = s->read_only || s->write_err || verify_read;
722		✗	int64_t inner_pos = read_only ? s->pos : block_pos;
723		✗	if (s->inner_pos != inner_pos) {
724		✗	inner_pos = ffurl_seek(s->inner, inner_pos, SEEK_SET);
725		✗	if (inner_pos < 0) {
726		✗	av_log(h, AV_LOG_ERROR, "Failed to seek underlying protocol: %s\n",
727		✗	av_err2str(inner_pos));
728		✗	if (!read_only) {
729			/* Release pending state to avoid stalling other threads. Don't
730			* mark this as failed, since the seek error may be unrelated to
731			* the block and should probably be tried again. */
732		✗	atomic_compare_exchange_strong_explicit(&block->state, &state,
733			BLOCK_NONE,
734			memory_order_relaxed,
735			memory_order_relaxed);
736			}
737		✗	return inner_pos;
738			}
739
740		✗	av_log(h, AV_LOG_DEBUG, "Inner seek to 0x%"PRIx64"\n", inner_pos);
741		✗	s->inner_pos = inner_pos;
742			}
743
744		✗	if (read_only) {
745			/* Directly defer to the underlying protocol */
746		✗	ret = ffurl_read(s->inner, buf, size);
747		✗	if (ret < 0)
748		✗	return ret;
749
750			/* Verify the read data against the cached data if requested */
751		✗	if (verify_read && memcmp(buf, tmp, ret)) {
752		✗	av_log(h, AV_LOG_ERROR, "Cache verification failed for %d bytes "
753			"in block 0x%"PRIx64" at offset 0x%"PRIx64" + %"PRId64"!\n",
754			ret, block_id, block_pos, offset);
755			}
756
757		✗	s->pos = s->inner_pos = inner_pos + ret;
758		✗	return ret;
759			}
760
761		✗	int write_back = 1;
762		✗	if (s->cache_data && !is_race) {
763			/* Read directly into memory mapped cache file */
764		✗	tmp = s->cache_data + block_pos;
765		✗	write_back = 0;
766		✗	} else if (size >= block_size && !offset) {
767			/* Read directly into output buffer if aligned and large enough */
768		✗	tmp = buf;
769			} else {
770			/* Read into temporary buffer and copy later */
771		✗	tmp = s->tmp_buf;
772			}
773
774			/* Try and fetch the entire block */
775		✗	av_assert0(inner_pos == block_pos);
776		✗	int bytes_read = 0;
777		✗	while (bytes_read < block_size) {
778		✗	ret = ffurl_read(s->inner, &tmp[bytes_read], block_size - bytes_read);
779		✗	if (!ret || ret == AVERROR_EOF)
780			break;
781		✗	else if (ret < 0) {
782		✗	av_log(h, AV_LOG_ERROR, "Failed to read block 0x%"PRIx64": %s\n",
783		✗	block_id, av_err2str(ret));
784		✗	int new_state = BLOCK_FAILED;
785		✗	if (ret == AVERROR(EAGAIN) || ret == AVERROR_EXIT)
786		✗	new_state = BLOCK_NONE; /* transient error, allow retries */
787
788			/* Try to mark block as failed; ignore errors - any mismatch
789			* here will mean that either another thread already marked it
790			* as failed, or successfully cached it in the meantime */
791		✗	atomic_compare_exchange_strong_explicit(&block->state, &state,
792			new_state,
793			memory_order_relaxed,
794			memory_order_relaxed);
795		✗	return ret;
796			}
797
798		✗	bytes_read += ret;
799		✗	s->inner_pos += ret;
800			}
801
802		✗	if (bytes_read < block_size) {
803			/* Learned location of true EOF, update filesize */
804		✗	ret = set_filesize(h, inner_pos + bytes_read);
805		✗	if (ret < 0)
806		✗	return ret;
807			}
808
809		✗	if (bytes_read > 0) {
810		✗	ret = write_back ? write_cache(s, tmp, bytes_read, block_pos) : 0;
811		✗	if (ret < 0) {
812		✗	av_log(h, AV_LOG_ERROR, "Failed to write to cache file: %s\n",
813		✗	av_err2str(ret));
814		✗	s->write_err = 1;
815			/* Mark as NONE, not FAILED, since the block itself is fine -
816			* just absent from the cache. */
817		✗	atomic_compare_exchange_strong_explicit(&block->state, &state,
818			BLOCK_NONE,
819			memory_order_relaxed,
820			memory_order_relaxed);
821			} else {
822		✗	uint32_t crc = get_block_crc(tmp, bytes_read);
823		✗	av_log(h, AV_LOG_TRACE, "Cached %d bytes to block 0x%"PRIx64" at "
824			"offset 0x%"PRIx64", CRC 0x%08X\n", bytes_read, block_id,
825			block_pos, crc);
826		✗	atomic_store_explicit(&block->state, crc, memory_order_release);
827			}
828			} else {
829		✗	return AVERROR_EOF;
830			}
831
832		✗	size = FFMIN(bytes_read - offset, size);
833		✗	if (size <= 0)
834		✗	return AVERROR_EOF;
835		✗	if (tmp != buf)
836		✗	memcpy(buf, &tmp[offset], size);
837		✗	s->pos += size;
838		✗	return size;
839			}
840
841		✗	static int64_t shared_seek(URLContext *h, int64_t pos, int whence)
842			{
843		✗	SharedContext *s = h->priv_data;
844		✗	const int64_t filesize = get_filesize(h);
845			int64_t res;
846
847		✗	switch (whence) {
848		✗	case AVSEEK_SIZE:
849		✗	if (filesize)
850		✗	return filesize;
851		✗	res = ffurl_seek(s->inner, pos, whence);
852		✗	if (res > 0) {
853		✗	if (set_filesize(h, res) < 0)
854		✗	return AVERROR(EINVAL);
855			}
856		✗	return res;
857		✗	case SEEK_SET:
858		✗	break;
859		✗	case SEEK_CUR:
860		✗	pos += s->pos;
861		✗	break;
862		✗	case SEEK_END:
863		✗	if (filesize) {
864		✗	pos += filesize;
865		✗	break;
866			}
867			/* Defer to underlying protocol if filesize is unknown */
868		✗	res = ffurl_seek(s->inner, pos, whence);
869		✗	if (res < 0)
870		✗	return res;
871			/* Opportunistically update known filesize */
872		✗	if (set_filesize(h, res - pos) < 0)
873		✗	return AVERROR(EINVAL);
874		✗	av_log(h, AV_LOG_DEBUG, "Inner seek to 0x%"PRIx64"\n", res);
875		✗	return s->pos = s->inner_pos = res;
876		✗	default:
877		✗	return AVERROR(EINVAL);
878			}
879
880		✗	if (pos < 0)
881		✗	return AVERROR(EINVAL);
882
883		✗	av_log(h, AV_LOG_DEBUG, "Virtual seek to 0x%"PRIx64"\n", pos);
884		✗	return s->pos = pos;
885			}
886
887		✗	static int shared_get_file_handle(URLContext *h)
888			{
889		✗	SharedContext *s = h->priv_data;
890		✗	return ffurl_get_file_handle(s->inner);
891			}
892
893		✗	static int shared_get_short_seek(URLContext *h)
894			{
895		✗	SharedContext *s = h->priv_data;
896		✗	int ret = ffurl_get_short_seek(s->inner);
897		✗	return ret > 0 ? FFMAX(ret, s->block_size) : s->block_size;
898			}
899
900			#define OFFSET(x) offsetof(SharedContext, x)
901			#define D AV_OPT_FLAG_DECODING_PARAM
902
903			static const AVOption options[] = {
904			{ "cache_dir", "Directory path for shared file cache", OFFSET(cache_dir), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D },
905			{ "block_shift", "Set the base 2 logarithm of the block size", OFFSET(block_shift), AV_OPT_TYPE_INT, {.i64 = 15}, 9, 30, .flags = D },
906			{ "read_only", "Don't write data to the cache, only read from it", OFFSET(read_only), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
907			{ "cache_verify", "Verify correctness of the cache against the source", OFFSET(verify), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
908			{ "cache_timeout", "Time in us to wait before re-fetching pending blocks", OFFSET(timeout), AV_OPT_TYPE_INT64, {.i64 = 10000}, 0, INT64_MAX, .flags = D },
909			{ "retry_errors", "Re-request blocks even if they previously failed", OFFSET(retry_errors), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
910			{ "retry_corrupt", "Re-request blocks that fail the CRC check", OFFSET(retry_corrupt), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
911			{0},
912			};
913
914			static const AVClass shared_context_class = {
915			.class_name = "shared",
916			.item_name = av_default_item_name,
917			.option = options,
918			.version = LIBAVUTIL_VERSION_INT,
919			};
920
921			const URLProtocol ff_shared_protocol = {
922			.name = "shared",
923			.url_open2 = shared_open,
924			.url_read = shared_read,
925			.url_seek = shared_seek,
926			.url_close = shared_close,
927			.url_get_file_handle = shared_get_file_handle,
928			.url_get_short_seek = shared_get_short_seek,
929			.priv_data_size = sizeof(SharedContext),
930			.priv_data_class = &shared_context_class,
931			};
932