From 28f206a09d7241eda81c705ca741d35c9f164783 Mon Sep 17 00:00:00 2001 From: Andrei Alexeyev Date: Thu, 5 Jan 2023 01:17:54 +0100 Subject: [PATCH] vfs,build: Add "resource index" VFS backend This is currently unused. It's going to be the backbone for a new on-the-fly resource fetching system for the Emscripten port. In this mode, the virtual resource directory structure is embedded into the executable. Files are referenced by their "content IDs", which are currently their sha256 hashes. The "resindex" VFS backend allows one to define a custom file open function, which connects the content IDs to actual data. A example implementation is provided in resindex_layered, which simply opens the content ID as a file under a specified VFS path. --- resources/meson.build | 52 +++++- scripts/index-resources.py | 164 ++++++++++++++++++ scripts/meson.build | 6 + scripts/res-index-install.py | 62 +++++++ src/vfs/meson.build | 9 + src/vfs/resindex.c | 277 ++++++++++++++++++++++++++++++ src/vfs/resindex.h | 46 +++++ src/vfs/resindex_layered.c | 38 ++++ src/vfs/resindex_layered.h | 14 ++ src/vfs/resindex_layered_public.h | 15 ++ 10 files changed, 681 insertions(+), 2 deletions(-) create mode 100755 scripts/index-resources.py create mode 100755 scripts/res-index-install.py create mode 100644 src/vfs/resindex.c create mode 100644 src/vfs/resindex.h create mode 100644 src/vfs/resindex_layered.c create mode 100644 src/vfs/resindex_layered.h create mode 100644 src/vfs/resindex_layered_public.h diff --git a/resources/meson.build b/resources/meson.build index f11dc695..9aaab63f 100644 --- a/resources/meson.build +++ b/resources/meson.build @@ -5,6 +5,56 @@ packages = [ '00-taisei', ] +use_static_res_index = false + +foreach pkg : packages + pkg_pkgdir = '@0@.pkgdir'.format(pkg) + subdir(pkg_pkgdir) +endforeach + +if use_static_res_index + resindex_deps = [] + resindex_cmd = [ + index_resources_command, + '@OUTPUT@', + ] + + foreach pkg : packages + pkg_pkgdir = '@0@.pkgdir'.format(pkg) + resindex_cmd += [resources_dir / pkg_pkgdir] + + if transpile_glsl + resindex_cmd += [meson.current_build_dir() / pkg_pkgdir] + endif + endforeach + + if transpile_glsl + resindex_deps += essl_targets + resindex_cmd += [ + '--exclude', '**/*.spv', + '--exclude', '**/*.glslh', + ] + endif + + resindex_cmd += [ + '--exclude', '**/*.build', + '--depfile', '@DEPFILE@', + ] + + resindex = custom_target( + command : resindex_cmd, + depfile : 'res-index.inc.h.d', + output : 'res-index.inc.h', + depends : resindex_deps, + build_by_default : false, + ) + + taisei_deps += declare_dependency(include_directories : include_directories('.')) + + meson.add_install_script(res_index_install_command, resindex, data_path) + subdir_done() +endif + if host_machine.system() == 'emscripten' em_data_prefix = '/@0@'.format(config.get_unquoted('TAISEI_BUILDCONF_DATA_PATH')) em_bundles = ['gfx', 'misc'] @@ -57,8 +107,6 @@ foreach pkg : packages pkg_zip = '@0@.zip'.format(pkg) pkg_path = join_paths(meson.current_source_dir(), pkg_pkgdir) - subdir(pkg_pkgdir) - if host_machine.system() == 'emscripten' foreach bundle : em_bundles var_patterns = 'em_bundle_@0@_patterns'.format(bundle) diff --git a/scripts/index-resources.py b/scripts/index-resources.py new file mode 100755 index 00000000..6f11635f --- /dev/null +++ b/scripts/index-resources.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 + +import hashlib + +from pathlib import Path + +from taiseilib.common import ( + DirPathType, + add_common_args, + run_main, + write_depfile, + update_text_file, +) + + +def quote(s): + return '"{}"'.format(s.encode('unicode_escape').decode('latin-1').replace('"', '\\"')) + + +class DirEntry: + def __init__(self, name): + self.name = name + self.subdirs = [] + self.subdirs_range = (0, 0) + self.files = [] + self.files_range = (0, 0) + + +class FileEntry: + def __init__(self, path): + self.path = path + self.name = path.name + + def unique_id(self): + return hashlib.sha256(self.path.read_bytes()).hexdigest() + + +def make_range(begin, end): + if begin == end: + return 0, 0 + return begin, end - begin + + +def index(args): + deps = [str(Path(__file__).resolve())] + + def excluded(path): + return path.name[0] == '.' or any(path.match(x) for x in args.exclude) + + def make_union_map(dirs): + u = {} + + for d in dirs: + u.update(dict((p.relative_to(d), p) for p in d.glob('**/*') if not excluded(p))) + + return u + + pathmap = make_union_map(args.directories) + import collections + nestedmap_factory = lambda: collections.defaultdict(nestedmap_factory) + nestedmap = nestedmap_factory() + + # Yes, it is probably stupid to build a nested data structure only to recursively scan it immediately, but it was the path of least resistance when I rewrote this crap last time and I don't care. + + for relpath, realpath in sorted(pathmap.items()): + if not realpath.is_file(): + continue + + parts = relpath.parts + target = nestedmap + for part in relpath.parts[:-1]: + target = target[part] + target[relpath.name] = realpath + + rootent = DirEntry(None) + + def scan(node, name): + dirent = DirEntry(name) + + for key, val in sorted(node.items()): + if isinstance(val, dict): + dirent.subdirs.append(scan(val, key)) + else: + dirent.files.append(FileEntry(val)) + deps.append(val) + + return dirent + + rootent.subdirs.append(scan(nestedmap, None)) + + lines = [] + out = lines.append + + ordered_dirs = [] + next_scan = [rootent] + + while next_scan: + this_scan = tuple(next_scan) + next_scan = [] + for dirent in this_scan: + begin = len(ordered_dirs) + for subdir in dirent.subdirs: + ordered_dirs.append(subdir) + end = len(ordered_dirs) + dirent.subdirs_range = make_range(begin, end) + next_scan += ordered_dirs[begin:end] + + findex = 0 + for n, dirent in enumerate(ordered_dirs): + dirent.files_range = make_range(findex, findex + len(dirent.files)) + + out("DIR({index:5}, {name}, {subdirs_ofs}, {subdirs_num}, {files_ofs}, {files_num})".format( + index=n, + name=(quote(dirent.name) if dirent.name is not None else "NULL"), + subdirs_ofs=dirent.subdirs_range[0], + subdirs_num=dirent.subdirs_range[1], + files_ofs=dirent.files_range[0], + files_num=dirent.files_range[1])) + + for fn, fentry in enumerate(dirent.files): + out("FILE({index:4}, {unique_id}, {name}, {source_path})".format( + index=findex + fn, + name=quote(fentry.name), + unique_id=quote(fentry.unique_id()), + source_path=quote(str(fentry.path.resolve())))) + + findex += len(dirent.files) + + update_text_file(args.output, '\n'.join(lines)) + + if args.depfile is not None: + write_depfile(args.depfile, args.output, deps) + + +def main(args): + import argparse + parser = argparse.ArgumentParser(description='Generate a static resource index with sha256 sums', prog=args[0]) + + parser.add_argument('output', + type=Path, + help='the output index file path' + ) + + parser.add_argument('directories', + metavar='directory', + nargs='+', + type=DirPathType, + help='the directory to index' + ) + + parser.add_argument('--exclude', + action='append', + default=[], + help='file exclusion pattern' + ) + + add_common_args(parser, depfile=True) + + args = parser.parse_args(args[1:]) + index(args) + + +if __name__ == '__main__': + run_main(main) diff --git a/scripts/meson.build b/scripts/meson.build index a97909bd..b333a5fe 100644 --- a/scripts/meson.build +++ b/scripts/meson.build @@ -136,3 +136,9 @@ format_array_command = [format_array_script] on_dist_script = find_program(files('on-meson-dist.py')) meson.add_dist_script(on_dist_script) + +index_resources_script = find_program(files('index-resources.py')) +index_resources_command = [index_resources_script, common_taiseilib_args] + +res_index_install_script = find_program(files('res-index-install.py')) +res_index_install_command = [res_index_install_script, common_taiseilib_args] diff --git a/scripts/res-index-install.py b/scripts/res-index-install.py new file mode 100755 index 00000000..65e51812 --- /dev/null +++ b/scripts/res-index-install.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +import sys +import os +import shutil + +from ast import literal_eval +from pathlib import Path + +from taiseilib.common import ( + add_common_args, + run_main, +) + + +def install(args): + install_root = Path(os.environ['MESON_INSTALL_DESTDIR_PREFIX']) + install_dir = install_root / args.subdir + install_dir.mkdir(exist_ok=True, parents=True) + installed_files = set(p.name for p in install_dir.iterdir()) + quiet = bool(os.environ.get('MESON_INSTALL_QUIET', False)) + + for line in args.index.read_text().split('\n'): + if not line.startswith('FILE('): + continue + + _, unique_id, _, srcpath = literal_eval(line[4:]) + + if unique_id in installed_files: + continue + + dstpath = install_dir / unique_id + assert not dstpath.exists() + + if not quiet: + print('Installing resource {} as {}'.format(srcpath, dstpath)) + + shutil.copy2(srcpath, dstpath) + installed_files.add(unique_id) + + +def main(args): + import argparse + parser = argparse.ArgumentParser(description='Install resources from res-index', prog=args[0]) + + parser.add_argument('index', + type=Path, + help='the index file' + ) + + parser.add_argument('subdir', + help='the target subdirectory (inside prefix)' + ) + + add_common_args(parser) + + args = parser.parse_args(args[1:]) + install(args) + + +if __name__ == '__main__': + run_main(main) diff --git a/src/vfs/meson.build b/src/vfs/meson.build index 2f9436eb..e4777cf9 100644 --- a/src/vfs/meson.build +++ b/src/vfs/meson.build @@ -50,5 +50,14 @@ else ) endif +if use_static_res_index + vfs_src += files('resindex.c') + vfs_src += resindex + + if is_developer_build + vfs_src += files('resindex_layered.c') + endif +endif + subdir('platform_paths') vfs_src += vfs_platform_paths_src diff --git a/src/vfs/resindex.c b/src/vfs/resindex.c new file mode 100644 index 00000000..7723b318 --- /dev/null +++ b/src/vfs/resindex.c @@ -0,0 +1,277 @@ +/* + * This software is licensed under the terms of the MIT License. + * See COPYING for further information. + * --- + * Copyright (c) 2011-2019, Lukas Weber . + * Copyright (c) 2012-2019, Andrei Alexeyev . +*/ + +#include "taisei.h" + +#include "resindex.h" + +VFS_NODE_TYPE(VFSResIndexNode, { + void *index_entry; + VFSResIndexFSContext *context; + struct VFSResIndexNode *root_node; +}); + +static const RIdxDirEntry ridx_dirs[] = { + #define DIR(_idx, _name, _subdirs_ofs, _subdirs_num, _files_ofs, _files_num) \ + [_idx] = { _name, _subdirs_ofs, _subdirs_num, _files_ofs, _files_num }, + #define FILE(...) + #include "res-index.inc.h" + #undef DIR + #undef FILE +}; + +static const RIdxFileEntry ridx_files[] = { + #define DIR(...) + #define FILE(_idx, _id, _name, _srcpath) \ + [_idx] = { _name, _id }, + #include "res-index.inc.h" + #undef DIR + #undef FILE +}; + +#define RIDX_IS_DIR(p) \ + ((RIdxDirEntry*)(p) >= ridx_dirs && (RIdxDirEntry*)(p) <= ridx_dirs + ARRAY_SIZE(ridx_dirs)) + +#define RIDX_IS_FILE(p) \ + ((RIdxFileEntry*)(p) >= ridx_files && (RIdxFileEntry*)(p) <= ridx_files + ARRAY_SIZE(ridx_files)) + +#define RIDX_AS_DIR(p) ({ \ + assert(RIDX_IS_DIR(p)); \ + CASTPTR_ASSUME_ALIGNED(p, RIdxDirEntry); \ +}) + +#define RIDX_AS_FILE(p) ({ \ + assert(RIDX_IS_FILE(p)); \ + CASTPTR_ASSUME_ALIGNED(p, RIdxFileEntry); \ +}) + +uint resindex_num_dir_entries(void) { + return ARRAY_SIZE(ridx_dirs); +} + +const RIdxDirEntry *resindex_get_dir_entry(uint idx) { + assert(idx < ARRAY_SIZE(ridx_dirs)); + return ridx_dirs + idx; +} + +uint resindex_num_file_entries(void) { + return ARRAY_SIZE(ridx_files); +} + +const RIdxFileEntry *resindex_get_file_entry(uint idx) { + assert(idx < ARRAY_SIZE(ridx_files)); + return ridx_files + idx; +} + +INLINE const char *ridx_dir_name(const RIdxDirEntry *d, const char *rootname) { + if(d == &ridx_dirs[0]) { + assert(d->name == NULL); + return rootname; + } + + return NOT_NULL(d->name); +} + +attr_unused +INLINE const char *ridx_node_name(VFSResIndexNode *rinode) { + if(RIDX_IS_DIR(rinode->index_entry)) { + return ridx_dir_name(RIDX_AS_DIR(rinode->index_entry), ""); + } + return RIDX_AS_FILE(rinode->index_entry)->name; +} + +INLINE bool ridx_node_is_root(VFSResIndexNode *rinode) { + assert(rinode->root_node == rinode); + return rinode->index_entry == &ridx_dirs[0]; +} + +static const RIdxDirEntry *ridx_subdir_lookup(const RIdxDirEntry *parent, const char *name) { + const RIdxDirEntry *begin = ridx_dirs + parent->subdirs_ofs; + const RIdxDirEntry *end = begin + parent->subdirs_num; + + for(const RIdxDirEntry *d = begin; d < end; ++d) { + if(!strcmp(name, NOT_NULL(d->name))) { + return d; + } + } + + return NULL; +} + +static const RIdxFileEntry *ridx_file_lookup(const RIdxDirEntry *parent, const char *name) { + const RIdxFileEntry *begin = ridx_files + parent->files_ofs; + const RIdxFileEntry *end = begin + parent->files_num; + + for(const RIdxFileEntry *f = begin; f < end; ++f) { + if(!strcmp(name, f->name)) { + return f; + } + } + + return NULL; +} + +static VFSResIndexNode *ridx_alloc_node(VFSResIndexNode *parent, void *content); + +static VFSInfo vfs_resindex_query(VFSNode *node) { + return (VFSInfo) { + .exists = true, + .is_dir = RIDX_IS_DIR(VFS_NODE_CAST(VFSResIndexNode, node)->index_entry), + .is_readonly = true, + }; +} + +static VFSNode *vfs_resindex_locate(VFSNode *root, const char *path) { + assert(*path); + + auto rindoe = VFS_NODE_CAST(VFSResIndexNode, root); + + if(!RIDX_IS_DIR(rindoe->index_entry)) { + vfs_set_error("Not a directory: %s", RIDX_AS_FILE(rindoe->index_entry)->name); + return NULL; + } + + const RIdxDirEntry *parent = RIDX_AS_DIR(rindoe->index_entry); + + char path_copy[strlen(path) + 1], *lpath, *rpath; + memcpy(path_copy, path, sizeof(path_copy)); + vfs_path_split_left(path_copy, &lpath, &rpath); + + for(;;) { + void *result = (void*)ridx_subdir_lookup(parent, lpath); + + if(!result) { + result = (void*)ridx_file_lookup(parent, lpath); + + if(*rpath && result) { + // non-final component is a file + vfs_set_error("Not a directory: %s", RIDX_AS_FILE(result)->name); + return NULL; + } + } + + if(!result) { + vfs_set_error("No such file or directory: %s", lpath); + return NULL; + } + + if(!*rpath) { + return &ridx_alloc_node(rindoe, result)->as_generic; + } + + parent = result; + vfs_path_split_left(rpath, &lpath, &rpath); + } +} + +static const char *vfs_resindex_iter(VFSNode *node, void **opaque) { + const RIdxDirEntry *dir; + + auto rindoe = VFS_NODE_CAST(VFSResIndexNode, node); + + if(!RIDX_IS_DIR(rindoe->index_entry)) { + return NULL; + } + + dir = RIDX_AS_DIR(rindoe->index_entry); + intptr_t *iter = (intptr_t *)opaque; + assert(*iter >= 0); + + int i = *iter; + *iter = i + 1; + + if(i < dir->subdirs_num) { + return ridx_dirs[dir->subdirs_ofs + i].name; + } + + if(i - dir->subdirs_num < dir->files_num) { + return ridx_files[dir->files_ofs + i - dir->subdirs_num].name; + } + + return NULL; +} + +static char *vfs_resindex_repr(VFSNode *node) { + auto rinode = VFS_NODE_CAST(VFSResIndexNode, node); + + if(RIDX_IS_DIR(rinode->index_entry)) { + const RIdxDirEntry *d = RIDX_AS_DIR(rinode->index_entry); + return strfmt( + "resource index directory #%i (%s)", + (int)(d - ridx_dirs), ridx_dir_name(d, "") + ); + } else { + const RIdxFileEntry *f = RIDX_AS_FILE(rinode->index_entry); + return strfmt( + "resource index file #%i: %s (%s)", + (int)(f - ridx_files), NOT_NULL(f->content_id), NOT_NULL(f->name) + ); + } +} + +static void vfs_resindex_free(VFSNode *node) { + auto rinode = VFS_NODE_CAST(VFSResIndexNode, node); + + if(ridx_node_is_root(rinode)) { + VFSResIndexFSContext *ctx = NOT_NULL(rinode->index_entry); + NOT_NULL(ctx->procs.free)(ctx); + } else { + vfs_decref(rinode->root_node); + } +} + +static SDL_RWops *vfs_resindex_open(VFSNode *node, VFSOpenMode mode) { + if(mode & VFS_MODE_WRITE) { + vfs_set_error("Read-only filesystem"); + return NULL; + } + + auto rinode = VFS_NODE_CAST(VFSResIndexNode, node); + + if(RIDX_IS_DIR(rinode->root_node)) { + vfs_set_error("Is a directory: %s", RIDX_AS_DIR(rinode->root_node)->name); + return NULL; + } + + const RIdxFileEntry *f = RIDX_AS_FILE(rinode->index_entry); + VFSResIndexFSContext *ctx = NOT_NULL(rinode->context); + return NOT_NULL(ctx->procs.open)(ctx, NOT_NULL(f->content_id), mode); +} + +VFS_NODE_FUNCS(VFSResIndexNode, { + .free = vfs_resindex_free, + .iter = vfs_resindex_iter, + .locate = vfs_resindex_locate, + .open = vfs_resindex_open, + .query = vfs_resindex_query, + .repr = vfs_resindex_repr, +}); + +static VFSResIndexNode *ridx_alloc_node(VFSResIndexNode *parent, void *content) { + auto root = NOT_NULL(parent->root_node); + assert(ridx_node_is_root(root)); + vfs_incref(root); + + return VFS_ALLOC(VFSResIndexNode, { + .root_node = root, + .index_entry = content, + .context = NOT_NULL(parent->context), + }); +} + +VFSNode *vfs_resindex_create(VFSResIndexFSContext *ctx) { + auto n = VFS_ALLOC(VFSResIndexNode, { + .context = ctx, + .index_entry = (void*)&ridx_dirs[0], + }); + + n->root_node = n; + assert(ridx_node_is_root(n)); + + return &n->as_generic; +} diff --git a/src/vfs/resindex.h b/src/vfs/resindex.h new file mode 100644 index 00000000..41b1ec84 --- /dev/null +++ b/src/vfs/resindex.h @@ -0,0 +1,46 @@ +/* + * This software is licensed under the terms of the MIT License. + * See COPYING for further information. + * --- + * Copyright (c) 2011-2019, Lukas Weber . + * Copyright (c) 2012-2019, Andrei Alexeyev . +*/ + +#pragma once +#include "taisei.h" + +#include "private.h" + +typedef struct VFSResIndexFSContext VFSResIndexFSContext; + +typedef SDL_RWops *(*VFSResIndexFSOpenProc)(VFSResIndexFSContext *ctx, const char *content_id, VFSOpenMode mode); +typedef void (*VFSResIndexFSFreeProc)(VFSResIndexFSContext *ctx); + +typedef struct VFSResIndexFSContext { + struct { + VFSResIndexFSOpenProc open; + VFSResIndexFSFreeProc free; + } procs; + void *userdata; +} VFSResIndexFSContext; + +VFSNode *vfs_resindex_create(VFSResIndexFSContext *ctx); + +typedef struct RIdxDirEntry { + const char *name; + int subdirs_ofs; + int subdirs_num; + int files_ofs; + int files_num; +} RIdxDirEntry; + +typedef struct RIdxFileEntry { + const char *name; + const char *content_id; +} RIdxFileEntry; + +uint resindex_num_dir_entries(void); +const RIdxDirEntry *resindex_get_dir_entry(uint idx); + +uint resindex_num_file_entries(void); +const RIdxFileEntry *resindex_get_file_entry(uint idx); diff --git a/src/vfs/resindex_layered.c b/src/vfs/resindex_layered.c new file mode 100644 index 00000000..cb55c227 --- /dev/null +++ b/src/vfs/resindex_layered.c @@ -0,0 +1,38 @@ +/* + * This software is licensed under the terms of the MIT License. + * See COPYING for further information. + * --- + * Copyright (c) 2011-2019, Lukas Weber . + * Copyright (c) 2012-2019, Andrei Alexeyev . +*/ + +#include "taisei.h" + +#include "resindex.h" +#include "resindex_layered.h" +#include "resindex_layered_public.h" + +static SDL_RWops *vfs_resindex_layered_open(VFSResIndexFSContext *ctx, const char *content_id, VFSOpenMode mode) { + const char *backend_path = ctx->userdata; + size_t pathlen = strlen(backend_path) + 1 + strlen(content_id); + char path[pathlen + 1]; + snprintf(path, sizeof(path), "%s" VFS_PATH_SEPARATOR_STR "%s", backend_path, content_id); + return vfs_open(path, mode); +} + +static void vfs_resindex_layered_free(VFSResIndexFSContext *ctx) { + mem_free(ctx->userdata); + mem_free(ctx); +} + +VFSNode *vfs_resindex_layered_create(const char *backend_vfspath) { + auto ctx = ALLOC(VFSResIndexFSContext); + ctx->procs.open = vfs_resindex_layered_open; + ctx->procs.free = vfs_resindex_layered_free; + ctx->userdata = vfs_path_normalize_alloc(backend_vfspath); + return vfs_resindex_create(ctx); +} + +bool vfs_mount_resindex_layered(const char *mountpoint, const char *backend_vfspath) { + return vfs_mount_or_decref(vfs_root, mountpoint, vfs_resindex_layered_create(backend_vfspath)); +} diff --git a/src/vfs/resindex_layered.h b/src/vfs/resindex_layered.h new file mode 100644 index 00000000..c3a810be --- /dev/null +++ b/src/vfs/resindex_layered.h @@ -0,0 +1,14 @@ +/* + * This software is licensed under the terms of the MIT License. + * See COPYING for further information. + * --- + * Copyright (c) 2011-2019, Lukas Weber . + * Copyright (c) 2012-2019, Andrei Alexeyev . +*/ + +#pragma once +#include "taisei.h" + +#include "private.h" + +VFSNode *vfs_resindex_layered_create(const char *backend_vfspath); diff --git a/src/vfs/resindex_layered_public.h b/src/vfs/resindex_layered_public.h new file mode 100644 index 00000000..4d4ed649 --- /dev/null +++ b/src/vfs/resindex_layered_public.h @@ -0,0 +1,15 @@ +/* + * This software is licensed under the terms of the MIT License. + * See COPYING for further information. + * --- + * Copyright (c) 2011-2019, Lukas Weber . + * Copyright (c) 2012-2019, Andrei Alexeyev . +*/ + +#pragma once +#include "taisei.h" + +#include "public.h" + +bool vfs_mount_resindex_layered(const char *mountpoint, const char *backend_vfspath) + attr_nonnull(1, 2) attr_nodiscard;