diff --git a/resources/meson.build b/resources/meson.build index f11dc695..9aaab63f 100644 --- a/resources/meson.build +++ b/resources/meson.build @@ -5,6 +5,56 @@ packages = [ '00-taisei', ] +use_static_res_index = false + +foreach pkg : packages + pkg_pkgdir = '@0@.pkgdir'.format(pkg) + subdir(pkg_pkgdir) +endforeach + +if use_static_res_index + resindex_deps = [] + resindex_cmd = [ + index_resources_command, + '@OUTPUT@', + ] + + foreach pkg : packages + pkg_pkgdir = '@0@.pkgdir'.format(pkg) + resindex_cmd += [resources_dir / pkg_pkgdir] + + if transpile_glsl + resindex_cmd += [meson.current_build_dir() / pkg_pkgdir] + endif + endforeach + + if transpile_glsl + resindex_deps += essl_targets + resindex_cmd += [ + '--exclude', '**/*.spv', + '--exclude', '**/*.glslh', + ] + endif + + resindex_cmd += [ + '--exclude', '**/*.build', + '--depfile', '@DEPFILE@', + ] + + resindex = custom_target( + command : resindex_cmd, + depfile : 'res-index.inc.h.d', + output : 'res-index.inc.h', + depends : resindex_deps, + build_by_default : false, + ) + + taisei_deps += declare_dependency(include_directories : include_directories('.')) + + meson.add_install_script(res_index_install_command, resindex, data_path) + subdir_done() +endif + if host_machine.system() == 'emscripten' em_data_prefix = '/@0@'.format(config.get_unquoted('TAISEI_BUILDCONF_DATA_PATH')) em_bundles = ['gfx', 'misc'] @@ -57,8 +107,6 @@ foreach pkg : packages pkg_zip = '@0@.zip'.format(pkg) pkg_path = join_paths(meson.current_source_dir(), pkg_pkgdir) - subdir(pkg_pkgdir) - if host_machine.system() == 'emscripten' foreach bundle : em_bundles var_patterns = 'em_bundle_@0@_patterns'.format(bundle) diff --git a/scripts/index-resources.py b/scripts/index-resources.py new file mode 100755 index 00000000..6f11635f --- /dev/null +++ b/scripts/index-resources.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 + +import hashlib + +from pathlib import Path + +from taiseilib.common import ( + DirPathType, + add_common_args, + run_main, + write_depfile, + update_text_file, +) + + +def quote(s): + return '"{}"'.format(s.encode('unicode_escape').decode('latin-1').replace('"', '\\"')) + + +class DirEntry: + def __init__(self, name): + self.name = name + self.subdirs = [] + self.subdirs_range = (0, 0) + self.files = [] + self.files_range = (0, 0) + + +class FileEntry: + def __init__(self, path): + self.path = path + self.name = path.name + + def unique_id(self): + return hashlib.sha256(self.path.read_bytes()).hexdigest() + + +def make_range(begin, end): + if begin == end: + return 0, 0 + return begin, end - begin + + +def index(args): + deps = [str(Path(__file__).resolve())] + + def excluded(path): + return path.name[0] == '.' or any(path.match(x) for x in args.exclude) + + def make_union_map(dirs): + u = {} + + for d in dirs: + u.update(dict((p.relative_to(d), p) for p in d.glob('**/*') if not excluded(p))) + + return u + + pathmap = make_union_map(args.directories) + import collections + nestedmap_factory = lambda: collections.defaultdict(nestedmap_factory) + nestedmap = nestedmap_factory() + + # Yes, it is probably stupid to build a nested data structure only to recursively scan it immediately, but it was the path of least resistance when I rewrote this crap last time and I don't care. + + for relpath, realpath in sorted(pathmap.items()): + if not realpath.is_file(): + continue + + parts = relpath.parts + target = nestedmap + for part in relpath.parts[:-1]: + target = target[part] + target[relpath.name] = realpath + + rootent = DirEntry(None) + + def scan(node, name): + dirent = DirEntry(name) + + for key, val in sorted(node.items()): + if isinstance(val, dict): + dirent.subdirs.append(scan(val, key)) + else: + dirent.files.append(FileEntry(val)) + deps.append(val) + + return dirent + + rootent.subdirs.append(scan(nestedmap, None)) + + lines = [] + out = lines.append + + ordered_dirs = [] + next_scan = [rootent] + + while next_scan: + this_scan = tuple(next_scan) + next_scan = [] + for dirent in this_scan: + begin = len(ordered_dirs) + for subdir in dirent.subdirs: + ordered_dirs.append(subdir) + end = len(ordered_dirs) + dirent.subdirs_range = make_range(begin, end) + next_scan += ordered_dirs[begin:end] + + findex = 0 + for n, dirent in enumerate(ordered_dirs): + dirent.files_range = make_range(findex, findex + len(dirent.files)) + + out("DIR({index:5}, {name}, {subdirs_ofs}, {subdirs_num}, {files_ofs}, {files_num})".format( + index=n, + name=(quote(dirent.name) if dirent.name is not None else "NULL"), + subdirs_ofs=dirent.subdirs_range[0], + subdirs_num=dirent.subdirs_range[1], + files_ofs=dirent.files_range[0], + files_num=dirent.files_range[1])) + + for fn, fentry in enumerate(dirent.files): + out("FILE({index:4}, {unique_id}, {name}, {source_path})".format( + index=findex + fn, + name=quote(fentry.name), + unique_id=quote(fentry.unique_id()), + source_path=quote(str(fentry.path.resolve())))) + + findex += len(dirent.files) + + update_text_file(args.output, '\n'.join(lines)) + + if args.depfile is not None: + write_depfile(args.depfile, args.output, deps) + + +def main(args): + import argparse + parser = argparse.ArgumentParser(description='Generate a static resource index with sha256 sums', prog=args[0]) + + parser.add_argument('output', + type=Path, + help='the output index file path' + ) + + parser.add_argument('directories', + metavar='directory', + nargs='+', + type=DirPathType, + help='the directory to index' + ) + + parser.add_argument('--exclude', + action='append', + default=[], + help='file exclusion pattern' + ) + + add_common_args(parser, depfile=True) + + args = parser.parse_args(args[1:]) + index(args) + + +if __name__ == '__main__': + run_main(main) diff --git a/scripts/meson.build b/scripts/meson.build index a97909bd..b333a5fe 100644 --- a/scripts/meson.build +++ b/scripts/meson.build @@ -136,3 +136,9 @@ format_array_command = [format_array_script] on_dist_script = find_program(files('on-meson-dist.py')) meson.add_dist_script(on_dist_script) + +index_resources_script = find_program(files('index-resources.py')) +index_resources_command = [index_resources_script, common_taiseilib_args] + +res_index_install_script = find_program(files('res-index-install.py')) +res_index_install_command = [res_index_install_script, common_taiseilib_args] diff --git a/scripts/res-index-install.py b/scripts/res-index-install.py new file mode 100755 index 00000000..65e51812 --- /dev/null +++ b/scripts/res-index-install.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +import sys +import os +import shutil + +from ast import literal_eval +from pathlib import Path + +from taiseilib.common import ( + add_common_args, + run_main, +) + + +def install(args): + install_root = Path(os.environ['MESON_INSTALL_DESTDIR_PREFIX']) + install_dir = install_root / args.subdir + install_dir.mkdir(exist_ok=True, parents=True) + installed_files = set(p.name for p in install_dir.iterdir()) + quiet = bool(os.environ.get('MESON_INSTALL_QUIET', False)) + + for line in args.index.read_text().split('\n'): + if not line.startswith('FILE('): + continue + + _, unique_id, _, srcpath = literal_eval(line[4:]) + + if unique_id in installed_files: + continue + + dstpath = install_dir / unique_id + assert not dstpath.exists() + + if not quiet: + print('Installing resource {} as {}'.format(srcpath, dstpath)) + + shutil.copy2(srcpath, dstpath) + installed_files.add(unique_id) + + +def main(args): + import argparse + parser = argparse.ArgumentParser(description='Install resources from res-index', prog=args[0]) + + parser.add_argument('index', + type=Path, + help='the index file' + ) + + parser.add_argument('subdir', + help='the target subdirectory (inside prefix)' + ) + + add_common_args(parser) + + args = parser.parse_args(args[1:]) + install(args) + + +if __name__ == '__main__': + run_main(main) diff --git a/src/vfs/meson.build b/src/vfs/meson.build index 2f9436eb..e4777cf9 100644 --- a/src/vfs/meson.build +++ b/src/vfs/meson.build @@ -50,5 +50,14 @@ else ) endif +if use_static_res_index + vfs_src += files('resindex.c') + vfs_src += resindex + + if is_developer_build + vfs_src += files('resindex_layered.c') + endif +endif + subdir('platform_paths') vfs_src += vfs_platform_paths_src diff --git a/src/vfs/resindex.c b/src/vfs/resindex.c new file mode 100644 index 00000000..7723b318 --- /dev/null +++ b/src/vfs/resindex.c @@ -0,0 +1,277 @@ +/* + * This software is licensed under the terms of the MIT License. + * See COPYING for further information. + * --- + * Copyright (c) 2011-2019, Lukas Weber . + * Copyright (c) 2012-2019, Andrei Alexeyev . +*/ + +#include "taisei.h" + +#include "resindex.h" + +VFS_NODE_TYPE(VFSResIndexNode, { + void *index_entry; + VFSResIndexFSContext *context; + struct VFSResIndexNode *root_node; +}); + +static const RIdxDirEntry ridx_dirs[] = { + #define DIR(_idx, _name, _subdirs_ofs, _subdirs_num, _files_ofs, _files_num) \ + [_idx] = { _name, _subdirs_ofs, _subdirs_num, _files_ofs, _files_num }, + #define FILE(...) + #include "res-index.inc.h" + #undef DIR + #undef FILE +}; + +static const RIdxFileEntry ridx_files[] = { + #define DIR(...) + #define FILE(_idx, _id, _name, _srcpath) \ + [_idx] = { _name, _id }, + #include "res-index.inc.h" + #undef DIR + #undef FILE +}; + +#define RIDX_IS_DIR(p) \ + ((RIdxDirEntry*)(p) >= ridx_dirs && (RIdxDirEntry*)(p) <= ridx_dirs + ARRAY_SIZE(ridx_dirs)) + +#define RIDX_IS_FILE(p) \ + ((RIdxFileEntry*)(p) >= ridx_files && (RIdxFileEntry*)(p) <= ridx_files + ARRAY_SIZE(ridx_files)) + +#define RIDX_AS_DIR(p) ({ \ + assert(RIDX_IS_DIR(p)); \ + CASTPTR_ASSUME_ALIGNED(p, RIdxDirEntry); \ +}) + +#define RIDX_AS_FILE(p) ({ \ + assert(RIDX_IS_FILE(p)); \ + CASTPTR_ASSUME_ALIGNED(p, RIdxFileEntry); \ +}) + +uint resindex_num_dir_entries(void) { + return ARRAY_SIZE(ridx_dirs); +} + +const RIdxDirEntry *resindex_get_dir_entry(uint idx) { + assert(idx < ARRAY_SIZE(ridx_dirs)); + return ridx_dirs + idx; +} + +uint resindex_num_file_entries(void) { + return ARRAY_SIZE(ridx_files); +} + +const RIdxFileEntry *resindex_get_file_entry(uint idx) { + assert(idx < ARRAY_SIZE(ridx_files)); + return ridx_files + idx; +} + +INLINE const char *ridx_dir_name(const RIdxDirEntry *d, const char *rootname) { + if(d == &ridx_dirs[0]) { + assert(d->name == NULL); + return rootname; + } + + return NOT_NULL(d->name); +} + +attr_unused +INLINE const char *ridx_node_name(VFSResIndexNode *rinode) { + if(RIDX_IS_DIR(rinode->index_entry)) { + return ridx_dir_name(RIDX_AS_DIR(rinode->index_entry), ""); + } + return RIDX_AS_FILE(rinode->index_entry)->name; +} + +INLINE bool ridx_node_is_root(VFSResIndexNode *rinode) { + assert(rinode->root_node == rinode); + return rinode->index_entry == &ridx_dirs[0]; +} + +static const RIdxDirEntry *ridx_subdir_lookup(const RIdxDirEntry *parent, const char *name) { + const RIdxDirEntry *begin = ridx_dirs + parent->subdirs_ofs; + const RIdxDirEntry *end = begin + parent->subdirs_num; + + for(const RIdxDirEntry *d = begin; d < end; ++d) { + if(!strcmp(name, NOT_NULL(d->name))) { + return d; + } + } + + return NULL; +} + +static const RIdxFileEntry *ridx_file_lookup(const RIdxDirEntry *parent, const char *name) { + const RIdxFileEntry *begin = ridx_files + parent->files_ofs; + const RIdxFileEntry *end = begin + parent->files_num; + + for(const RIdxFileEntry *f = begin; f < end; ++f) { + if(!strcmp(name, f->name)) { + return f; + } + } + + return NULL; +} + +static VFSResIndexNode *ridx_alloc_node(VFSResIndexNode *parent, void *content); + +static VFSInfo vfs_resindex_query(VFSNode *node) { + return (VFSInfo) { + .exists = true, + .is_dir = RIDX_IS_DIR(VFS_NODE_CAST(VFSResIndexNode, node)->index_entry), + .is_readonly = true, + }; +} + +static VFSNode *vfs_resindex_locate(VFSNode *root, const char *path) { + assert(*path); + + auto rindoe = VFS_NODE_CAST(VFSResIndexNode, root); + + if(!RIDX_IS_DIR(rindoe->index_entry)) { + vfs_set_error("Not a directory: %s", RIDX_AS_FILE(rindoe->index_entry)->name); + return NULL; + } + + const RIdxDirEntry *parent = RIDX_AS_DIR(rindoe->index_entry); + + char path_copy[strlen(path) + 1], *lpath, *rpath; + memcpy(path_copy, path, sizeof(path_copy)); + vfs_path_split_left(path_copy, &lpath, &rpath); + + for(;;) { + void *result = (void*)ridx_subdir_lookup(parent, lpath); + + if(!result) { + result = (void*)ridx_file_lookup(parent, lpath); + + if(*rpath && result) { + // non-final component is a file + vfs_set_error("Not a directory: %s", RIDX_AS_FILE(result)->name); + return NULL; + } + } + + if(!result) { + vfs_set_error("No such file or directory: %s", lpath); + return NULL; + } + + if(!*rpath) { + return &ridx_alloc_node(rindoe, result)->as_generic; + } + + parent = result; + vfs_path_split_left(rpath, &lpath, &rpath); + } +} + +static const char *vfs_resindex_iter(VFSNode *node, void **opaque) { + const RIdxDirEntry *dir; + + auto rindoe = VFS_NODE_CAST(VFSResIndexNode, node); + + if(!RIDX_IS_DIR(rindoe->index_entry)) { + return NULL; + } + + dir = RIDX_AS_DIR(rindoe->index_entry); + intptr_t *iter = (intptr_t *)opaque; + assert(*iter >= 0); + + int i = *iter; + *iter = i + 1; + + if(i < dir->subdirs_num) { + return ridx_dirs[dir->subdirs_ofs + i].name; + } + + if(i - dir->subdirs_num < dir->files_num) { + return ridx_files[dir->files_ofs + i - dir->subdirs_num].name; + } + + return NULL; +} + +static char *vfs_resindex_repr(VFSNode *node) { + auto rinode = VFS_NODE_CAST(VFSResIndexNode, node); + + if(RIDX_IS_DIR(rinode->index_entry)) { + const RIdxDirEntry *d = RIDX_AS_DIR(rinode->index_entry); + return strfmt( + "resource index directory #%i (%s)", + (int)(d - ridx_dirs), ridx_dir_name(d, "") + ); + } else { + const RIdxFileEntry *f = RIDX_AS_FILE(rinode->index_entry); + return strfmt( + "resource index file #%i: %s (%s)", + (int)(f - ridx_files), NOT_NULL(f->content_id), NOT_NULL(f->name) + ); + } +} + +static void vfs_resindex_free(VFSNode *node) { + auto rinode = VFS_NODE_CAST(VFSResIndexNode, node); + + if(ridx_node_is_root(rinode)) { + VFSResIndexFSContext *ctx = NOT_NULL(rinode->index_entry); + NOT_NULL(ctx->procs.free)(ctx); + } else { + vfs_decref(rinode->root_node); + } +} + +static SDL_RWops *vfs_resindex_open(VFSNode *node, VFSOpenMode mode) { + if(mode & VFS_MODE_WRITE) { + vfs_set_error("Read-only filesystem"); + return NULL; + } + + auto rinode = VFS_NODE_CAST(VFSResIndexNode, node); + + if(RIDX_IS_DIR(rinode->root_node)) { + vfs_set_error("Is a directory: %s", RIDX_AS_DIR(rinode->root_node)->name); + return NULL; + } + + const RIdxFileEntry *f = RIDX_AS_FILE(rinode->index_entry); + VFSResIndexFSContext *ctx = NOT_NULL(rinode->context); + return NOT_NULL(ctx->procs.open)(ctx, NOT_NULL(f->content_id), mode); +} + +VFS_NODE_FUNCS(VFSResIndexNode, { + .free = vfs_resindex_free, + .iter = vfs_resindex_iter, + .locate = vfs_resindex_locate, + .open = vfs_resindex_open, + .query = vfs_resindex_query, + .repr = vfs_resindex_repr, +}); + +static VFSResIndexNode *ridx_alloc_node(VFSResIndexNode *parent, void *content) { + auto root = NOT_NULL(parent->root_node); + assert(ridx_node_is_root(root)); + vfs_incref(root); + + return VFS_ALLOC(VFSResIndexNode, { + .root_node = root, + .index_entry = content, + .context = NOT_NULL(parent->context), + }); +} + +VFSNode *vfs_resindex_create(VFSResIndexFSContext *ctx) { + auto n = VFS_ALLOC(VFSResIndexNode, { + .context = ctx, + .index_entry = (void*)&ridx_dirs[0], + }); + + n->root_node = n; + assert(ridx_node_is_root(n)); + + return &n->as_generic; +} diff --git a/src/vfs/resindex.h b/src/vfs/resindex.h new file mode 100644 index 00000000..41b1ec84 --- /dev/null +++ b/src/vfs/resindex.h @@ -0,0 +1,46 @@ +/* + * This software is licensed under the terms of the MIT License. + * See COPYING for further information. + * --- + * Copyright (c) 2011-2019, Lukas Weber . + * Copyright (c) 2012-2019, Andrei Alexeyev . +*/ + +#pragma once +#include "taisei.h" + +#include "private.h" + +typedef struct VFSResIndexFSContext VFSResIndexFSContext; + +typedef SDL_RWops *(*VFSResIndexFSOpenProc)(VFSResIndexFSContext *ctx, const char *content_id, VFSOpenMode mode); +typedef void (*VFSResIndexFSFreeProc)(VFSResIndexFSContext *ctx); + +typedef struct VFSResIndexFSContext { + struct { + VFSResIndexFSOpenProc open; + VFSResIndexFSFreeProc free; + } procs; + void *userdata; +} VFSResIndexFSContext; + +VFSNode *vfs_resindex_create(VFSResIndexFSContext *ctx); + +typedef struct RIdxDirEntry { + const char *name; + int subdirs_ofs; + int subdirs_num; + int files_ofs; + int files_num; +} RIdxDirEntry; + +typedef struct RIdxFileEntry { + const char *name; + const char *content_id; +} RIdxFileEntry; + +uint resindex_num_dir_entries(void); +const RIdxDirEntry *resindex_get_dir_entry(uint idx); + +uint resindex_num_file_entries(void); +const RIdxFileEntry *resindex_get_file_entry(uint idx); diff --git a/src/vfs/resindex_layered.c b/src/vfs/resindex_layered.c new file mode 100644 index 00000000..cb55c227 --- /dev/null +++ b/src/vfs/resindex_layered.c @@ -0,0 +1,38 @@ +/* + * This software is licensed under the terms of the MIT License. + * See COPYING for further information. + * --- + * Copyright (c) 2011-2019, Lukas Weber . + * Copyright (c) 2012-2019, Andrei Alexeyev . +*/ + +#include "taisei.h" + +#include "resindex.h" +#include "resindex_layered.h" +#include "resindex_layered_public.h" + +static SDL_RWops *vfs_resindex_layered_open(VFSResIndexFSContext *ctx, const char *content_id, VFSOpenMode mode) { + const char *backend_path = ctx->userdata; + size_t pathlen = strlen(backend_path) + 1 + strlen(content_id); + char path[pathlen + 1]; + snprintf(path, sizeof(path), "%s" VFS_PATH_SEPARATOR_STR "%s", backend_path, content_id); + return vfs_open(path, mode); +} + +static void vfs_resindex_layered_free(VFSResIndexFSContext *ctx) { + mem_free(ctx->userdata); + mem_free(ctx); +} + +VFSNode *vfs_resindex_layered_create(const char *backend_vfspath) { + auto ctx = ALLOC(VFSResIndexFSContext); + ctx->procs.open = vfs_resindex_layered_open; + ctx->procs.free = vfs_resindex_layered_free; + ctx->userdata = vfs_path_normalize_alloc(backend_vfspath); + return vfs_resindex_create(ctx); +} + +bool vfs_mount_resindex_layered(const char *mountpoint, const char *backend_vfspath) { + return vfs_mount_or_decref(vfs_root, mountpoint, vfs_resindex_layered_create(backend_vfspath)); +} diff --git a/src/vfs/resindex_layered.h b/src/vfs/resindex_layered.h new file mode 100644 index 00000000..c3a810be --- /dev/null +++ b/src/vfs/resindex_layered.h @@ -0,0 +1,14 @@ +/* + * This software is licensed under the terms of the MIT License. + * See COPYING for further information. + * --- + * Copyright (c) 2011-2019, Lukas Weber . + * Copyright (c) 2012-2019, Andrei Alexeyev . +*/ + +#pragma once +#include "taisei.h" + +#include "private.h" + +VFSNode *vfs_resindex_layered_create(const char *backend_vfspath); diff --git a/src/vfs/resindex_layered_public.h b/src/vfs/resindex_layered_public.h new file mode 100644 index 00000000..4d4ed649 --- /dev/null +++ b/src/vfs/resindex_layered_public.h @@ -0,0 +1,15 @@ +/* + * This software is licensed under the terms of the MIT License. + * See COPYING for further information. + * --- + * Copyright (c) 2011-2019, Lukas Weber . + * Copyright (c) 2012-2019, Andrei Alexeyev . +*/ + +#pragma once +#include "taisei.h" + +#include "public.h" + +bool vfs_mount_resindex_layered(const char *mountpoint, const char *backend_vfspath) + attr_nonnull(1, 2) attr_nodiscard;