vfs,build: Add "resource index" VFS backend

This is currently unused. It's going to be the backbone for a new
on-the-fly resource fetching system for the Emscripten port.

In this mode, the virtual resource directory structure is embedded into
the executable. Files are referenced by their "content IDs", which are
currently their sha256 hashes. The "resindex" VFS backend allows one to
define a custom file open function, which connects the content IDs to
actual data. A example implementation is provided in resindex_layered,
which simply opens the content ID as a file under a specified VFS path.
This commit is contained in:
Andrei Alexeyev 2023-01-05 01:17:54 +01:00
parent 84b67a66bf
commit 28f206a09d
No known key found for this signature in database
GPG key ID: 72D26128040B9690
10 changed files with 681 additions and 2 deletions

View file

@ -5,6 +5,56 @@ packages = [
'00-taisei',
]
use_static_res_index = false
foreach pkg : packages
pkg_pkgdir = '@0@.pkgdir'.format(pkg)
subdir(pkg_pkgdir)
endforeach
if use_static_res_index
resindex_deps = []
resindex_cmd = [
index_resources_command,
'@OUTPUT@',
]
foreach pkg : packages
pkg_pkgdir = '@0@.pkgdir'.format(pkg)
resindex_cmd += [resources_dir / pkg_pkgdir]
if transpile_glsl
resindex_cmd += [meson.current_build_dir() / pkg_pkgdir]
endif
endforeach
if transpile_glsl
resindex_deps += essl_targets
resindex_cmd += [
'--exclude', '**/*.spv',
'--exclude', '**/*.glslh',
]
endif
resindex_cmd += [
'--exclude', '**/*.build',
'--depfile', '@DEPFILE@',
]
resindex = custom_target(
command : resindex_cmd,
depfile : 'res-index.inc.h.d',
output : 'res-index.inc.h',
depends : resindex_deps,
build_by_default : false,
)
taisei_deps += declare_dependency(include_directories : include_directories('.'))
meson.add_install_script(res_index_install_command, resindex, data_path)
subdir_done()
endif
if host_machine.system() == 'emscripten'
em_data_prefix = '/@0@'.format(config.get_unquoted('TAISEI_BUILDCONF_DATA_PATH'))
em_bundles = ['gfx', 'misc']
@ -57,8 +107,6 @@ foreach pkg : packages
pkg_zip = '@0@.zip'.format(pkg)
pkg_path = join_paths(meson.current_source_dir(), pkg_pkgdir)
subdir(pkg_pkgdir)
if host_machine.system() == 'emscripten'
foreach bundle : em_bundles
var_patterns = 'em_bundle_@0@_patterns'.format(bundle)

164
scripts/index-resources.py Executable file
View file

@ -0,0 +1,164 @@
#!/usr/bin/env python3
import hashlib
from pathlib import Path
from taiseilib.common import (
DirPathType,
add_common_args,
run_main,
write_depfile,
update_text_file,
)
def quote(s):
return '"{}"'.format(s.encode('unicode_escape').decode('latin-1').replace('"', '\\"'))
class DirEntry:
def __init__(self, name):
self.name = name
self.subdirs = []
self.subdirs_range = (0, 0)
self.files = []
self.files_range = (0, 0)
class FileEntry:
def __init__(self, path):
self.path = path
self.name = path.name
def unique_id(self):
return hashlib.sha256(self.path.read_bytes()).hexdigest()
def make_range(begin, end):
if begin == end:
return 0, 0
return begin, end - begin
def index(args):
deps = [str(Path(__file__).resolve())]
def excluded(path):
return path.name[0] == '.' or any(path.match(x) for x in args.exclude)
def make_union_map(dirs):
u = {}
for d in dirs:
u.update(dict((p.relative_to(d), p) for p in d.glob('**/*') if not excluded(p)))
return u
pathmap = make_union_map(args.directories)
import collections
nestedmap_factory = lambda: collections.defaultdict(nestedmap_factory)
nestedmap = nestedmap_factory()
# Yes, it is probably stupid to build a nested data structure only to recursively scan it immediately, but it was the path of least resistance when I rewrote this crap last time and I don't care.
for relpath, realpath in sorted(pathmap.items()):
if not realpath.is_file():
continue
parts = relpath.parts
target = nestedmap
for part in relpath.parts[:-1]:
target = target[part]
target[relpath.name] = realpath
rootent = DirEntry(None)
def scan(node, name):
dirent = DirEntry(name)
for key, val in sorted(node.items()):
if isinstance(val, dict):
dirent.subdirs.append(scan(val, key))
else:
dirent.files.append(FileEntry(val))
deps.append(val)
return dirent
rootent.subdirs.append(scan(nestedmap, None))
lines = []
out = lines.append
ordered_dirs = []
next_scan = [rootent]
while next_scan:
this_scan = tuple(next_scan)
next_scan = []
for dirent in this_scan:
begin = len(ordered_dirs)
for subdir in dirent.subdirs:
ordered_dirs.append(subdir)
end = len(ordered_dirs)
dirent.subdirs_range = make_range(begin, end)
next_scan += ordered_dirs[begin:end]
findex = 0
for n, dirent in enumerate(ordered_dirs):
dirent.files_range = make_range(findex, findex + len(dirent.files))
out("DIR({index:5}, {name}, {subdirs_ofs}, {subdirs_num}, {files_ofs}, {files_num})".format(
index=n,
name=(quote(dirent.name) if dirent.name is not None else "NULL"),
subdirs_ofs=dirent.subdirs_range[0],
subdirs_num=dirent.subdirs_range[1],
files_ofs=dirent.files_range[0],
files_num=dirent.files_range[1]))
for fn, fentry in enumerate(dirent.files):
out("FILE({index:4}, {unique_id}, {name}, {source_path})".format(
index=findex + fn,
name=quote(fentry.name),
unique_id=quote(fentry.unique_id()),
source_path=quote(str(fentry.path.resolve()))))
findex += len(dirent.files)
update_text_file(args.output, '\n'.join(lines))
if args.depfile is not None:
write_depfile(args.depfile, args.output, deps)
def main(args):
import argparse
parser = argparse.ArgumentParser(description='Generate a static resource index with sha256 sums', prog=args[0])
parser.add_argument('output',
type=Path,
help='the output index file path'
)
parser.add_argument('directories',
metavar='directory',
nargs='+',
type=DirPathType,
help='the directory to index'
)
parser.add_argument('--exclude',
action='append',
default=[],
help='file exclusion pattern'
)
add_common_args(parser, depfile=True)
args = parser.parse_args(args[1:])
index(args)
if __name__ == '__main__':
run_main(main)

View file

@ -136,3 +136,9 @@ format_array_command = [format_array_script]
on_dist_script = find_program(files('on-meson-dist.py'))
meson.add_dist_script(on_dist_script)
index_resources_script = find_program(files('index-resources.py'))
index_resources_command = [index_resources_script, common_taiseilib_args]
res_index_install_script = find_program(files('res-index-install.py'))
res_index_install_command = [res_index_install_script, common_taiseilib_args]

62
scripts/res-index-install.py Executable file
View file

@ -0,0 +1,62 @@
#!/usr/bin/env python3
import sys
import os
import shutil
from ast import literal_eval
from pathlib import Path
from taiseilib.common import (
add_common_args,
run_main,
)
def install(args):
install_root = Path(os.environ['MESON_INSTALL_DESTDIR_PREFIX'])
install_dir = install_root / args.subdir
install_dir.mkdir(exist_ok=True, parents=True)
installed_files = set(p.name for p in install_dir.iterdir())
quiet = bool(os.environ.get('MESON_INSTALL_QUIET', False))
for line in args.index.read_text().split('\n'):
if not line.startswith('FILE('):
continue
_, unique_id, _, srcpath = literal_eval(line[4:])
if unique_id in installed_files:
continue
dstpath = install_dir / unique_id
assert not dstpath.exists()
if not quiet:
print('Installing resource {} as {}'.format(srcpath, dstpath))
shutil.copy2(srcpath, dstpath)
installed_files.add(unique_id)
def main(args):
import argparse
parser = argparse.ArgumentParser(description='Install resources from res-index', prog=args[0])
parser.add_argument('index',
type=Path,
help='the index file'
)
parser.add_argument('subdir',
help='the target subdirectory (inside prefix)'
)
add_common_args(parser)
args = parser.parse_args(args[1:])
install(args)
if __name__ == '__main__':
run_main(main)

View file

@ -50,5 +50,14 @@ else
)
endif
if use_static_res_index
vfs_src += files('resindex.c')
vfs_src += resindex
if is_developer_build
vfs_src += files('resindex_layered.c')
endif
endif
subdir('platform_paths')
vfs_src += vfs_platform_paths_src

277
src/vfs/resindex.c Normal file
View file

@ -0,0 +1,277 @@
/*
* This software is licensed under the terms of the MIT License.
* See COPYING for further information.
* ---
* Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
* Copyright (c) 2012-2019, Andrei Alexeyev <akari@taisei-project.org>.
*/
#include "taisei.h"
#include "resindex.h"
VFS_NODE_TYPE(VFSResIndexNode, {
void *index_entry;
VFSResIndexFSContext *context;
struct VFSResIndexNode *root_node;
});
static const RIdxDirEntry ridx_dirs[] = {
#define DIR(_idx, _name, _subdirs_ofs, _subdirs_num, _files_ofs, _files_num) \
[_idx] = { _name, _subdirs_ofs, _subdirs_num, _files_ofs, _files_num },
#define FILE(...)
#include "res-index.inc.h"
#undef DIR
#undef FILE
};
static const RIdxFileEntry ridx_files[] = {
#define DIR(...)
#define FILE(_idx, _id, _name, _srcpath) \
[_idx] = { _name, _id },
#include "res-index.inc.h"
#undef DIR
#undef FILE
};
#define RIDX_IS_DIR(p) \
((RIdxDirEntry*)(p) >= ridx_dirs && (RIdxDirEntry*)(p) <= ridx_dirs + ARRAY_SIZE(ridx_dirs))
#define RIDX_IS_FILE(p) \
((RIdxFileEntry*)(p) >= ridx_files && (RIdxFileEntry*)(p) <= ridx_files + ARRAY_SIZE(ridx_files))
#define RIDX_AS_DIR(p) ({ \
assert(RIDX_IS_DIR(p)); \
CASTPTR_ASSUME_ALIGNED(p, RIdxDirEntry); \
})
#define RIDX_AS_FILE(p) ({ \
assert(RIDX_IS_FILE(p)); \
CASTPTR_ASSUME_ALIGNED(p, RIdxFileEntry); \
})
uint resindex_num_dir_entries(void) {
return ARRAY_SIZE(ridx_dirs);
}
const RIdxDirEntry *resindex_get_dir_entry(uint idx) {
assert(idx < ARRAY_SIZE(ridx_dirs));
return ridx_dirs + idx;
}
uint resindex_num_file_entries(void) {
return ARRAY_SIZE(ridx_files);
}
const RIdxFileEntry *resindex_get_file_entry(uint idx) {
assert(idx < ARRAY_SIZE(ridx_files));
return ridx_files + idx;
}
INLINE const char *ridx_dir_name(const RIdxDirEntry *d, const char *rootname) {
if(d == &ridx_dirs[0]) {
assert(d->name == NULL);
return rootname;
}
return NOT_NULL(d->name);
}
attr_unused
INLINE const char *ridx_node_name(VFSResIndexNode *rinode) {
if(RIDX_IS_DIR(rinode->index_entry)) {
return ridx_dir_name(RIDX_AS_DIR(rinode->index_entry), "<root>");
}
return RIDX_AS_FILE(rinode->index_entry)->name;
}
INLINE bool ridx_node_is_root(VFSResIndexNode *rinode) {
assert(rinode->root_node == rinode);
return rinode->index_entry == &ridx_dirs[0];
}
static const RIdxDirEntry *ridx_subdir_lookup(const RIdxDirEntry *parent, const char *name) {
const RIdxDirEntry *begin = ridx_dirs + parent->subdirs_ofs;
const RIdxDirEntry *end = begin + parent->subdirs_num;
for(const RIdxDirEntry *d = begin; d < end; ++d) {
if(!strcmp(name, NOT_NULL(d->name))) {
return d;
}
}
return NULL;
}
static const RIdxFileEntry *ridx_file_lookup(const RIdxDirEntry *parent, const char *name) {
const RIdxFileEntry *begin = ridx_files + parent->files_ofs;
const RIdxFileEntry *end = begin + parent->files_num;
for(const RIdxFileEntry *f = begin; f < end; ++f) {
if(!strcmp(name, f->name)) {
return f;
}
}
return NULL;
}
static VFSResIndexNode *ridx_alloc_node(VFSResIndexNode *parent, void *content);
static VFSInfo vfs_resindex_query(VFSNode *node) {
return (VFSInfo) {
.exists = true,
.is_dir = RIDX_IS_DIR(VFS_NODE_CAST(VFSResIndexNode, node)->index_entry),
.is_readonly = true,
};
}
static VFSNode *vfs_resindex_locate(VFSNode *root, const char *path) {
assert(*path);
auto rindoe = VFS_NODE_CAST(VFSResIndexNode, root);
if(!RIDX_IS_DIR(rindoe->index_entry)) {
vfs_set_error("Not a directory: %s", RIDX_AS_FILE(rindoe->index_entry)->name);
return NULL;
}
const RIdxDirEntry *parent = RIDX_AS_DIR(rindoe->index_entry);
char path_copy[strlen(path) + 1], *lpath, *rpath;
memcpy(path_copy, path, sizeof(path_copy));
vfs_path_split_left(path_copy, &lpath, &rpath);
for(;;) {
void *result = (void*)ridx_subdir_lookup(parent, lpath);
if(!result) {
result = (void*)ridx_file_lookup(parent, lpath);
if(*rpath && result) {
// non-final component is a file
vfs_set_error("Not a directory: %s", RIDX_AS_FILE(result)->name);
return NULL;
}
}
if(!result) {
vfs_set_error("No such file or directory: %s", lpath);
return NULL;
}
if(!*rpath) {
return &ridx_alloc_node(rindoe, result)->as_generic;
}
parent = result;
vfs_path_split_left(rpath, &lpath, &rpath);
}
}
static const char *vfs_resindex_iter(VFSNode *node, void **opaque) {
const RIdxDirEntry *dir;
auto rindoe = VFS_NODE_CAST(VFSResIndexNode, node);
if(!RIDX_IS_DIR(rindoe->index_entry)) {
return NULL;
}
dir = RIDX_AS_DIR(rindoe->index_entry);
intptr_t *iter = (intptr_t *)opaque;
assert(*iter >= 0);
int i = *iter;
*iter = i + 1;
if(i < dir->subdirs_num) {
return ridx_dirs[dir->subdirs_ofs + i].name;
}
if(i - dir->subdirs_num < dir->files_num) {
return ridx_files[dir->files_ofs + i - dir->subdirs_num].name;
}
return NULL;
}
static char *vfs_resindex_repr(VFSNode *node) {
auto rinode = VFS_NODE_CAST(VFSResIndexNode, node);
if(RIDX_IS_DIR(rinode->index_entry)) {
const RIdxDirEntry *d = RIDX_AS_DIR(rinode->index_entry);
return strfmt(
"resource index directory #%i (%s)",
(int)(d - ridx_dirs), ridx_dir_name(d, "<root>")
);
} else {
const RIdxFileEntry *f = RIDX_AS_FILE(rinode->index_entry);
return strfmt(
"resource index file #%i: %s (%s)",
(int)(f - ridx_files), NOT_NULL(f->content_id), NOT_NULL(f->name)
);
}
}
static void vfs_resindex_free(VFSNode *node) {
auto rinode = VFS_NODE_CAST(VFSResIndexNode, node);
if(ridx_node_is_root(rinode)) {
VFSResIndexFSContext *ctx = NOT_NULL(rinode->index_entry);
NOT_NULL(ctx->procs.free)(ctx);
} else {
vfs_decref(rinode->root_node);
}
}
static SDL_RWops *vfs_resindex_open(VFSNode *node, VFSOpenMode mode) {
if(mode & VFS_MODE_WRITE) {
vfs_set_error("Read-only filesystem");
return NULL;
}
auto rinode = VFS_NODE_CAST(VFSResIndexNode, node);
if(RIDX_IS_DIR(rinode->root_node)) {
vfs_set_error("Is a directory: %s", RIDX_AS_DIR(rinode->root_node)->name);
return NULL;
}
const RIdxFileEntry *f = RIDX_AS_FILE(rinode->index_entry);
VFSResIndexFSContext *ctx = NOT_NULL(rinode->context);
return NOT_NULL(ctx->procs.open)(ctx, NOT_NULL(f->content_id), mode);
}
VFS_NODE_FUNCS(VFSResIndexNode, {
.free = vfs_resindex_free,
.iter = vfs_resindex_iter,
.locate = vfs_resindex_locate,
.open = vfs_resindex_open,
.query = vfs_resindex_query,
.repr = vfs_resindex_repr,
});
static VFSResIndexNode *ridx_alloc_node(VFSResIndexNode *parent, void *content) {
auto root = NOT_NULL(parent->root_node);
assert(ridx_node_is_root(root));
vfs_incref(root);
return VFS_ALLOC(VFSResIndexNode, {
.root_node = root,
.index_entry = content,
.context = NOT_NULL(parent->context),
});
}
VFSNode *vfs_resindex_create(VFSResIndexFSContext *ctx) {
auto n = VFS_ALLOC(VFSResIndexNode, {
.context = ctx,
.index_entry = (void*)&ridx_dirs[0],
});
n->root_node = n;
assert(ridx_node_is_root(n));
return &n->as_generic;
}

46
src/vfs/resindex.h Normal file
View file

@ -0,0 +1,46 @@
/*
* This software is licensed under the terms of the MIT License.
* See COPYING for further information.
* ---
* Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
* Copyright (c) 2012-2019, Andrei Alexeyev <akari@taisei-project.org>.
*/
#pragma once
#include "taisei.h"
#include "private.h"
typedef struct VFSResIndexFSContext VFSResIndexFSContext;
typedef SDL_RWops *(*VFSResIndexFSOpenProc)(VFSResIndexFSContext *ctx, const char *content_id, VFSOpenMode mode);
typedef void (*VFSResIndexFSFreeProc)(VFSResIndexFSContext *ctx);
typedef struct VFSResIndexFSContext {
struct {
VFSResIndexFSOpenProc open;
VFSResIndexFSFreeProc free;
} procs;
void *userdata;
} VFSResIndexFSContext;
VFSNode *vfs_resindex_create(VFSResIndexFSContext *ctx);
typedef struct RIdxDirEntry {
const char *name;
int subdirs_ofs;
int subdirs_num;
int files_ofs;
int files_num;
} RIdxDirEntry;
typedef struct RIdxFileEntry {
const char *name;
const char *content_id;
} RIdxFileEntry;
uint resindex_num_dir_entries(void);
const RIdxDirEntry *resindex_get_dir_entry(uint idx);
uint resindex_num_file_entries(void);
const RIdxFileEntry *resindex_get_file_entry(uint idx);

View file

@ -0,0 +1,38 @@
/*
* This software is licensed under the terms of the MIT License.
* See COPYING for further information.
* ---
* Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
* Copyright (c) 2012-2019, Andrei Alexeyev <akari@taisei-project.org>.
*/
#include "taisei.h"
#include "resindex.h"
#include "resindex_layered.h"
#include "resindex_layered_public.h"
static SDL_RWops *vfs_resindex_layered_open(VFSResIndexFSContext *ctx, const char *content_id, VFSOpenMode mode) {
const char *backend_path = ctx->userdata;
size_t pathlen = strlen(backend_path) + 1 + strlen(content_id);
char path[pathlen + 1];
snprintf(path, sizeof(path), "%s" VFS_PATH_SEPARATOR_STR "%s", backend_path, content_id);
return vfs_open(path, mode);
}
static void vfs_resindex_layered_free(VFSResIndexFSContext *ctx) {
mem_free(ctx->userdata);
mem_free(ctx);
}
VFSNode *vfs_resindex_layered_create(const char *backend_vfspath) {
auto ctx = ALLOC(VFSResIndexFSContext);
ctx->procs.open = vfs_resindex_layered_open;
ctx->procs.free = vfs_resindex_layered_free;
ctx->userdata = vfs_path_normalize_alloc(backend_vfspath);
return vfs_resindex_create(ctx);
}
bool vfs_mount_resindex_layered(const char *mountpoint, const char *backend_vfspath) {
return vfs_mount_or_decref(vfs_root, mountpoint, vfs_resindex_layered_create(backend_vfspath));
}

View file

@ -0,0 +1,14 @@
/*
* This software is licensed under the terms of the MIT License.
* See COPYING for further information.
* ---
* Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
* Copyright (c) 2012-2019, Andrei Alexeyev <akari@taisei-project.org>.
*/
#pragma once
#include "taisei.h"
#include "private.h"
VFSNode *vfs_resindex_layered_create(const char *backend_vfspath);

View file

@ -0,0 +1,15 @@
/*
* This software is licensed under the terms of the MIT License.
* See COPYING for further information.
* ---
* Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
* Copyright (c) 2012-2019, Andrei Alexeyev <akari@taisei-project.org>.
*/
#pragma once
#include "taisei.h"
#include "public.h"
bool vfs_mount_resindex_layered(const char *mountpoint, const char *backend_vfspath)
attr_nonnull(1, 2) attr_nodiscard;