shared.c File Reference

#include "libavutil/attributes.h"
#include "libavutil/avassert.h"
#include "libavutil/avstring.h"
#include "libavutil/crc.h"
#include "libavutil/error.h"
#include "libavutil/hash.h"
#include "libavutil/file_open.h"
#include "libavutil/mem.h"
#include "libavutil/opt.h"
#include "libavutil/time.h"
#include "internal.h"
#include "url.h"
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdatomic.h>
#include <string.h>
#include <sys/file.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>

Go to the source code of this file.

Data Structures
struct	Block
struct	Spacemap
struct	SharedContext

Macros
#define	HASH_METHOD   "SHA512/256"
	This hash should be resistant against collision attacks, so that an attacker could not generate e.g.
#define	HASH_SIZE   32
#define	HEADER_MAGIC   MKTAG(u'\xFF', 'S', 'h', '$')
#define	HEADER_VERSION   3
#define	MAX_CORRUPT_BLOCKS   10
	Hard watershed of consecutive failed blocks before we give up on the cache file altogether and assume it's entirely lost to us.
#define	DEF_SET_ONCE(ctype, atype)
#define	RELEASE_PENDING(block, state)
#define	OFFSET(x)
#define	D   AV_OPT_FLAG_DECODING_PARAM

Enumerations
enum	BlockState { BLOCK_NONE = 0 , BLOCK_PENDING , BLOCK_FAILED }

Functions
static int	hash_uri (uint8_t hash[HASH_SIZE], const char *uri)
static uint32_t	get_block_crc (const uint8_t *block, size_t block_size)
static int	shared_close (URLContext *h)
static int	cache_map (URLContext *h, int64_t filesize)
static int	spacemap_init (URLContext *h, const uint8_t hash[HASH_SIZE])
static int	spacemap_grow (URLContext *h, int64_t block)
static int64_t	get_filesize (URLContext *h)
static int	set_filesize (URLContext *h, int64_t new_size)
static int	is_ignorable_error (int64_t err)
static int	shared_open (URLContext *h, const char *arg, int flags, AVDictionary **options)
static int	spacemap_remap (URLContext *h, size_t map_size)
static int	read_cache (SharedContext *s, uint8_t *buf, size_t size, off_t offset)
static int	write_cache (SharedContext *s, const uint8_t *buf, size_t size, off_t offset)
static int	clamp_size (URLContext *h, int size, int64_t pos, int64_t filesize)
static int	shared_read (URLContext *h, unsigned char *buf, int size)
static int64_t	shared_seek (URLContext *h, int64_t pos, int whence)
static int	shared_get_file_handle (URLContext *h)
static int	shared_get_short_seek (URLContext *h)

Variables
static const AVOption	options []
static const AVClass	shared_context_class
const URLProtocol	ff_shared_protocol

Macro Definition Documentation

HASH_METHOD

#define HASH_METHOD   "SHA512/256"

This hash should be resistant against collision attacks, so that an attacker could not generate e.g.

two different URIs that map to the same cache file. This requires at least 64 bits of collision resistance in practice (i.e. 128 bits = 16 bytes of hash size). However, we can be conservative by computing e.g. a 256 bit hash and storing it inside the file header for verification.

Note that due to the way we use atomics, we should avoid zero bytes in the resulting hash; hence we tweak the input slightly to avoid this. The resulting loss in hash strength is negligible, since 32 bytes is already much more than needed.

Definition at line 62 of file shared.c.

Referenced by hash_uri().

HASH_SIZE

#define HASH_SIZE   32

Definition at line 63 of file shared.c.

Referenced by close_out(), hash_uri(), main(), shared_open(), and spacemap_init().

HEADER_MAGIC

#define HEADER_MAGIC   MKTAG(u'\xFF', 'S', 'h', '$')

Definition at line 64 of file shared.c.

Referenced by spacemap_init().

HEADER_VERSION

#define HEADER_VERSION   3

Definition at line 65 of file shared.c.

Referenced by hash_uri(), and spacemap_init().

MAX_CORRUPT_BLOCKS

#define MAX_CORRUPT_BLOCKS   10

Hard watershed of consecutive failed blocks before we give up on the cache file altogether and assume it's entirely lost to us.

Definition at line 71 of file shared.c.

Referenced by shared_read().

DEF_SET_ONCE

#define DEF_SET_ONCE	(		ctype,
			atype )

Value:

    static int set_once_##atype(atomic_##atype *const ptr, const ctype value)   \
    {                                                                           \
        ctype prev = 0;                                                         \
        av_assert1(value != 0);                                                 \
        if (atomic_compare_exchange_strong_explicit(                            \
                ptr, &prev, value, memory_order_release, memory_order_relaxed)) \
            return 1;                                                           \
        else if (prev == value)                                                 \
            return 0;                                                           \
        else                                                                    \
            return AVERROR(EINVAL);                                             \
    }

Definition at line 137 of file shared.c.

RELEASE_PENDING

#define RELEASE_PENDING	(		block,
			state )

Value:

    do {                                                                       \
        if (acquired) {                                                        \
            av_assert1(state == BLOCK_PENDING);                                \
            atomic_compare_exchange_strong_explicit(                           \
                &block->state, &state, BLOCK_NONE, memory_order_relaxed,       \
                memory_order_relaxed);                                         \
        }                                                                      \
    } while (0)

Referenced by shared_read().

OFFSET

#define OFFSET

(

x

)

Value:

offsetof(SharedContext, x)

Definition at line 1005 of file shared.c.

D

#define D   AV_OPT_FLAG_DECODING_PARAM

Definition at line 1006 of file shared.c.

Enumeration Type Documentation

BlockState

enum BlockState

Enumerator
BLOCK_NONE	block is not cached
BLOCK_PENDING	a thread is currently trying to write this block
BLOCK_FAILED	the underlying I/O source failed to read this block

Definition at line 93 of file shared.c.

Function Documentation

hash_uri()

int hash_uri ( uint8_t hash[HASH_SIZE], const char * uri )

static

Definition at line 73 of file shared.c.

Referenced by shared_open().

get_block_crc()

uint32_t get_block_crc ( const uint8_t * block, size_t block_size )

static

Definition at line 105 of file shared.c.

Referenced by shared_read().

shared_close()

int shared_close ( URLContext * h )

static

Definition at line 198 of file shared.c.

Referenced by shared_open().

cache_map()

int cache_map ( URLContext * h, int64_t filesize )

static

Definition at line 383 of file shared.c.

Referenced by set_filesize(), and shared_open().

spacemap_init()

int spacemap_init ( URLContext * h, const uint8_t hash[HASH_SIZE] )

static

Definition at line 525 of file shared.c.

Referenced by shared_open().

spacemap_grow()

int spacemap_grow ( URLContext * h, int64_t block )

static

Definition at line 489 of file shared.c.

Referenced by shared_open(), and shared_read().

get_filesize()

int64_t get_filesize ( URLContext * h )

static

Definition at line 224 of file shared.c.

Referenced by shared_open(), shared_read(), shared_seek(), and spacemap_grow().

set_filesize()

int set_filesize ( URLContext * h, int64_t new_size )

static

Definition at line 238 of file shared.c.

Referenced by shared_read(), and shared_seek().

is_ignorable_error()

int is_ignorable_error ( int64_t err )

static

Definition at line 261 of file shared.c.

Referenced by shared_open(), and shared_seek().

shared_open()

int shared_open ( URLContext * h, const char * arg, int flags, AVDictionary ** options )

static

Definition at line 276 of file shared.c.

spacemap_remap()

int spacemap_remap ( URLContext * h, size_t map_size )

static

Definition at line 419 of file shared.c.

Referenced by spacemap_grow(), and spacemap_init().

read_cache()

int read_cache ( SharedContext * s, uint8_t * buf, size_t size, off_t offset )

static

Definition at line 580 of file shared.c.

Referenced by shared_read().

write_cache()

int write_cache ( SharedContext * s, const uint8_t * buf, size_t size, off_t offset )

static

Definition at line 600 of file shared.c.

Referenced by shared_read().

clamp_size()

int clamp_size ( URLContext * h, int size, int64_t pos, int64_t filesize )

static

Definition at line 620 of file shared.c.

Referenced by shared_read().

shared_read()

int shared_read ( URLContext * h, unsigned char * buf, int size )

static

Definition at line 630 of file shared.c.

shared_seek()

int64_t shared_seek ( URLContext * h, int64_t pos, int whence )

static

Definition at line 929 of file shared.c.

shared_get_file_handle()

int shared_get_file_handle ( URLContext * h )

static

Definition at line 992 of file shared.c.

shared_get_short_seek()

int shared_get_short_seek ( URLContext * h )

static

Definition at line 998 of file shared.c.

Variable Documentation

options

const AVOption options[]

static

Initial value:

= {
    { "cache_dir",      "Directory path for shared file cache",             OFFSET(cache_dir),      AV_OPT_TYPE_STRING, {.str = NULL}, .flags = D },
    { "block_shift",    "Set the base 2 logarithm of the block size",       OFFSET(block_shift),    AV_OPT_TYPE_INT, {.i64 = 15}, 9, 30, .flags = D },
    { "read_only",      "Don't write data to the cache, only read from it", OFFSET(read_only),      AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
    { "cache_verify",   "Verify correctness of the cache against the source",   OFFSET(verify),     AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
    { "cache_timeout",  "Time in us to wait before re-fetching pending blocks", OFFSET(timeout),    AV_OPT_TYPE_INT64, {.i64 = 10000}, 0, INT64_MAX, .flags = D },
    { "ignore_errors",  "Continue even if the inner URL failed",            OFFSET(ignore_errors),  AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = D },
    { "retry_errors",   "Re-request blocks even if they previously failed", OFFSET(retry_errors),   AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
    { "retry_corrupt",  "Re-request blocks that fail the CRC check",        OFFSET(retry_corrupt),  AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, .flags = D },
    { "cache_size_max", "Limit the maximum amount of data cached",          OFFSET(cache_size_max), AV_OPT_TYPE_INT64, {.i64 = 0}, 0, INT64_MAX, .flags = D },
    {0},
}

Definition at line 1008 of file shared.c.

shared_context_class

const AVClass shared_context_class

static

Initial value:

= {
    .class_name = "shared",
    .item_name  = av_default_item_name,
    .option     = options,
    .version    = LIBAVUTIL_VERSION_INT,
}

Definition at line 1021 of file shared.c.

ff_shared_protocol

const URLProtocol ff_shared_protocol

Initial value:

= {
    .name                = "shared",
    .url_open2           = shared_open,
    .url_read            = shared_read,
    .url_seek            = shared_seek,
    .url_close           = shared_close,
    .url_get_file_handle = shared_get_file_handle,
    .url_get_short_seek  = shared_get_short_seek,
    .priv_data_size      = sizeof(SharedContext),
    .priv_data_class     = &shared_context_class,
}

Definition at line 1028 of file shared.c.