Implement seeking in zip files without reading everything to memory

In particular, this greatly reduces memory usage for music tracks.
This commit is contained in:
Andrei Alexeyev 2019-03-16 22:11:37 +02:00
parent c23c1239fe
commit cc6514151a
No known key found for this signature in database
GPG key ID: 363707CD4C7FE8A4
18 changed files with 342 additions and 56 deletions

View file

@ -21,7 +21,7 @@ Dependencies
- freetype2
- libpng >= 1.5.0
- libwebpdecoder >= 0.5 or libwebp >= 0.5
- libzip >= 1.0
- libzip >= 1.2
- zlib
Optional:

View file

@ -105,7 +105,7 @@ dep_sdl2 = dependency('sdl2', version : '>=2.0.5', required : t
dep_sdl2_mixer = dependency('SDL2_mixer', required : false, static : static)
dep_webp = dependency('libwebp', version : '>=0.5', required : false, static : static)
dep_webpdecoder = dependency('libwebpdecoder', version : '>=0.5', required : false, static : static)
dep_zip = dependency('libzip', version : '>=1.0', required : false, static : static)
dep_zip = dependency('libzip', version : '>=1.2', required : false, static : static)
dep_zlib = dependency('zlib', required : true, static : static)
dep_crypto = dependency('libcrypto', required : false, static : static)

View file

@ -0,0 +1 @@
^.*\.(ttf|png|webp|ogg|prog)$

View file

@ -2,6 +2,7 @@
import os
import sys
import re
from datetime import (
datetime,
@ -25,9 +26,21 @@ from taiseilib.common import (
def pack(args):
with ZipFile(str(args.output), 'w', ZIP_DEFLATED) as zf:
nocompress_file = args.directory / '.nocompress'
try:
nocompress = list(map(re.compile, filter(None, nocompress_file.read_text().strip().split('\n'))))
except FileNotFoundError:
nocompress = []
nocompress_file = None
zkwargs = {}
if (sys.version_info.major, sys.version_info.minor) >= (3, 7):
zkwargs['compresslevel'] = 9
with ZipFile(str(args.output), 'w', ZIP_DEFLATED, **zkwargs) as zf:
for path in sorted(args.directory.glob('**/*')):
if path.name == 'meson.build':
if path.name[0] == '.' or path.name == 'meson.build':
continue
relpath = path.relative_to(args.directory)
@ -38,11 +51,20 @@ def pack(args):
zi.external_attr = 0o40755 << 16 # drwxr-xr-x
zf.writestr(zi, '')
else:
zf.write(str(path), str(relpath))
ctype = ZIP_DEFLATED
for pattern in nocompress:
if pattern.match(str(relpath)):
ctype = ZIP_STORED
break
zf.write(str(path), str(relpath), compress_type=ctype)
if args.depfile is not None:
write_depfile(args.depfile, args.output,
[args.directory.resolve() / x for x in zf.namelist()] + [str(Path(__file__).resolve())]
[args.directory.resolve() / x for x in zf.namelist()] +
[str(Path(__file__).resolve())] +
list(filter(None, [nocompress_file]))
)

View file

@ -99,8 +99,7 @@ static void init_sdl(void) {
log_fatal("SDL_Init() failed: %s", SDL_GetError());
}
// initialize it
is_main_thread();
main_thread_id = SDL_ThreadID();
// * TODO: refine this and make it optional
// SDL_LogSetAllPriority(SDL_LOG_PRIORITY_DEBUG);

View file

@ -297,7 +297,6 @@ static FT_Face load_font_face(char *vfspath, long index) {
}
log_info("Loaded font '%s' (face %li)", syspath, index);
return face;
}

View file

@ -21,4 +21,8 @@
#include "rwops_zipfile.h"
#endif
#ifdef DEBUG
#include "rwops_trace.h"
#endif
#endif // IGUARD_rwops_all_h

View file

@ -13,3 +13,9 @@ if taisei_deps.contains(dep_zip)
'rwops_zipfile.c',
)
endif
if is_debug_build
rwops_src += files(
'rwops_trace.c',
)
endif

76
src/rwops/rwops_trace.c Normal file
View file

@ -0,0 +1,76 @@
/*
* This software is licensed under the terms of the MIT-License
* See COPYING for further information.
* ---
* Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
* Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
*/
#include "taisei.h"
#include "rwops_trace.h"
#include "util.h"
#define TRACE_SOURCE(rw) ((SDL_RWops*)((rw)->hidden.unknown.data1))
#define TRACE_AUTOCLOSE(rw) ((bool)((rw)->hidden.unknown.data2))
static int trace_close(SDL_RWops *rw) {
int ret = 0;
if(TRACE_AUTOCLOSE(rw)) {
ret = SDL_RWclose(TRACE_SOURCE(rw));
log_debug("[%lx :: %p] close() = %i", SDL_ThreadID(), (void*)rw, ret);
}
SDL_FreeRW(rw);
log_debug("[%lx :: %p] closed", SDL_ThreadID(), (void*)rw);
return ret;
}
static int64_t trace_seek(SDL_RWops *rw, int64_t offset, int whence) {
int64_t p = SDL_RWseek(TRACE_SOURCE(rw), offset, whence);
log_debug("[%lx :: %p] seek(offset=%"PRIi64"; whence=%i) = %"PRIi64, SDL_ThreadID(), (void*)rw, offset, whence, p);
return p;
}
static int64_t trace_size(SDL_RWops *rw) {
int64_t s = SDL_RWsize(TRACE_SOURCE(rw));
log_debug("[%lx :: %p] size() = %"PRIi64, SDL_ThreadID(), (void*)rw, s);
return s;
}
static size_t trace_read(SDL_RWops *rw, void *ptr, size_t size, size_t maxnum) {
size_t r = SDL_RWread(TRACE_SOURCE(rw), ptr, size, maxnum);
log_debug("[%lx :: %p] read(dest=%p; size=%zu; num=%zu) = %zu", SDL_ThreadID(), (void*)rw, ptr, size, maxnum, r);
log_debug("[%lx :: %p] `--> %"PRIi64, SDL_ThreadID(), (void*)rw, SDL_RWtell(TRACE_SOURCE(rw)));
if(size > 0 && maxnum > 0 && r == 0) {
// abort();
}
return r;
}
static size_t trace_write(SDL_RWops *rw, const void *ptr, size_t size, size_t maxnum) {
size_t w = SDL_RWwrite(TRACE_SOURCE(rw), ptr, size, maxnum);
log_debug("[%lx :: %p] write(dest=%p; size=%zu; num=%zu) = %zu", SDL_ThreadID(), (void*)rw, ptr, size, maxnum, w);
return w;
}
SDL_RWops *SDL_RWWrapTrace(SDL_RWops *src, bool autoclose) {
SDL_RWops *rw = SDL_AllocRW();
memset(rw, 0, sizeof(SDL_RWops));
rw->hidden.unknown.data1 = src;
rw->hidden.unknown.data2 = (void*)(intptr_t)autoclose;
rw->type = SDL_RWOPS_UNKNOWN;
rw->size = trace_size;
rw->seek = trace_seek;
rw->close = trace_close;
rw->read = trace_read;
rw->write = trace_write;
log_debug("[%lx :: %p] opened; src=%p", SDL_ThreadID(), (void*)rw, (void*)src);
return rw;
}

18
src/rwops/rwops_trace.h Normal file
View file

@ -0,0 +1,18 @@
/*
* This software is licensed under the terms of the MIT-License
* See COPYING for further information.
* ---
* Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
* Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
*/
#ifndef IGUARD_rwops_rwops_trace_h
#define IGUARD_rwops_rwops_trace_h
#include "taisei.h"
#include <SDL.h>
SDL_RWops *SDL_RWWrapTrace(SDL_RWops *src, bool autoclose);
#endif // IGUARD_rwops_rwops_trace_h

View file

@ -11,12 +11,55 @@
#include "rwops_zipfile.h"
#include "util.h"
typedef struct ZipRWData {
VFSNode *node;
zip_file_t *file;
int64_t pos;
} ZipRWData;
#define ZTLS(pdata) vfs_zipfile_get_tls((pdata)->zipnode, true)
static zip_file_t *ziprw_open(VFSZipPathData *pdata) {
zip_file_t *zipfile = zip_fopen_index(ZTLS(pdata)->zip, pdata->index, 0);
if(!zipfile) {
SDL_SetError("ZIP error: %s", zip_error_strerror(&ZTLS(pdata)->error));
}
return zipfile;
}
static zip_file_t *ziprw_get_zipfile(SDL_RWops *rw) {
ZipRWData *rwdata = rw->hidden.unknown.data1;
VFSZipPathData *pdata = rw->hidden.unknown.data2;
int64_t pos = rwdata->pos;
if(!vfs_zipfile_get_tls((pdata)->zipnode, false) && rwdata->file) {
zip_fclose(rwdata->file);
rwdata->file = NULL;
rwdata->pos = 0;
}
if(!rwdata->file) {
rwdata->file = ziprw_open(pdata);
SDL_RWseek(rw, pos, RW_SEEK_SET);
}
assert(rwdata->file);
return rwdata->file;
}
static int ziprw_close(SDL_RWops *rw) {
if(rw) {
if(rw->hidden.unknown.data2) {
zip_fclose(rw->hidden.unknown.data1);
ZipRWData *rwdata = rw->hidden.unknown.data1;
if(rwdata->file) {
zip_fclose(rwdata->file);
}
vfs_decref(rwdata->node);
free(rwdata);
SDL_FreeRW(rw);
}
@ -24,19 +67,119 @@ static int ziprw_close(SDL_RWops *rw) {
}
static int64_t ziprw_seek(SDL_RWops *rw, int64_t offset, int whence) {
SDL_SetError("Not implemented");
ZipRWData *rwdata = rw->hidden.unknown.data1;
if(!ziprw_get_zipfile(rw)) {
return -1;
}
if(zip_fseek(rwdata->file, offset, whence) == 0) {
return rwdata->pos = zip_ftell(rwdata->file);
}
SDL_SetError("ZIP error: %s", zip_error_strerror(zip_file_get_error(rwdata->file)));
return -1;
}
static int64_t ziprw_seek_emulated(SDL_RWops *rw, int64_t offset, int whence) {
ZipRWData *rwdata = rw->hidden.unknown.data1;
VFSZipPathData *pdata = rw->hidden.unknown.data2;
if(!ziprw_get_zipfile(rw)) {
return -1;
}
ssize_t new_pos;
ssize_t sz = SDL_RWsize(rw);
switch(whence) {
case RW_SEEK_SET: new_pos = offset; break;
case RW_SEEK_CUR: new_pos = rwdata->pos + offset; break;
case RW_SEEK_END:
if(sz < 0) {
return sz;
}
new_pos = sz - offset;
break;
}
if(new_pos < 0 || new_pos > sz) {
SDL_SetError("Seek offset out of range");
return -1;
}
if(new_pos > rwdata->pos) {
const size_t chunk_size = 4096;
do {
size_t read_size = imin(new_pos - rwdata->pos, chunk_size);
uint8_t chunk[read_size];
size_t actual_read_size = SDL_RWread(rw, chunk, 1, read_size);
assert(actual_read_size <= read_size);
if(actual_read_size < read_size) {
break;
}
} while(new_pos > rwdata->pos);
} else if(new_pos < rwdata->pos) {
zip_fclose(rwdata->file);
rwdata->file = ziprw_open(pdata);
rwdata->pos = 0;
int64_t s = ziprw_seek_emulated(rw, new_pos, RW_SEEK_SET);
assert(s == new_pos);
assert(s == rwdata->pos);
return s;
}
return rwdata->pos;
}
static int64_t ziprw_size(SDL_RWops *rw) {
SDL_SetError("Not implemented");
return -1;
VFSZipPathData *pdata = rw->hidden.unknown.data2;
if(pdata->size < 0) {
SDL_SetError("zip_stat_index() failed");
return -1;
}
return pdata->size;
}
static size_t ziprw_read(SDL_RWops *rw, void *ptr, size_t size, size_t maxnum) {
zip_file_t *zipfile = rw->hidden.unknown.data1;
// XXX: possible size overflow
return zip_fread(zipfile, ptr, size * maxnum);
ZipRWData *rwdata = rw->hidden.unknown.data1;
VFSZipPathData *pdata = rw->hidden.unknown.data2;
libzip_sucks:
if(!ziprw_get_zipfile(rw)) {
return 0;
}
size_t read_size = size * maxnum;
if(size != 0 && read_size / size != maxnum) {
SDL_SetError("Read size is too large");
return 0;
}
zip_int64_t bytes_read = zip_fread(rwdata->file, ptr, read_size);
if(bytes_read < 0) {
SDL_SetError("ZIP error: %s", zip_error_strerror(zip_file_get_error(rwdata->file)));
log_debug("ZIP error: %s", zip_error_strerror(zip_file_get_error(rwdata->file)));
return 0;
}
if(read_size > 0 && bytes_read == 0 && rw->seek == ziprw_seek && zip_ftell(rwdata->file) < pdata->size) {
log_debug("libzip BUG: EOF flag not cleared after seek, reopening file");
zip_fclose(rwdata->file);
rwdata->file = NULL;
goto libzip_sucks;
}
rwdata->pos += bytes_read;
return bytes_read / size;
}
static size_t ziprw_write(SDL_RWops *rw, const void *ptr, size_t size, size_t maxnum) {
@ -44,19 +187,29 @@ static size_t ziprw_write(SDL_RWops *rw, const void *ptr, size_t size, size_t ma
return -1;
}
SDL_RWops* SDL_RWFromZipFile(zip_file_t *zipfile, bool autoclose) {
SDL_RWops *SDL_RWFromZipFile(VFSNode *znode, VFSZipPathData *pdata) {
SDL_RWops *rw = SDL_AllocRW();
memset(rw, 0, sizeof(SDL_RWops));
rw->hidden.unknown.data1 = zipfile;
rw->hidden.unknown.data2 = (void*)(intptr_t)autoclose;
ZipRWData *rwdata = calloc(1, sizeof(*rwdata));
rwdata->node = znode;
vfs_incref(znode);
rw->hidden.unknown.data1 = rwdata;
rw->hidden.unknown.data2 = pdata;
rw->type = SDL_RWOPS_UNKNOWN;
rw->size = ziprw_size;
rw->seek = ziprw_seek;
rw->close = ziprw_close;
rw->read = ziprw_read;
rw->write = ziprw_write;
if(pdata->seekable) {
rw->seek = ziprw_seek;
} else {
rw->seek = ziprw_seek_emulated;
}
return rw;
}

View file

@ -13,7 +13,8 @@
#include <SDL.h>
#include <zip.h>
#include "vfs/zipfile_impl.h"
SDL_RWops* SDL_RWFromZipFile(zip_file_t *zipfile, bool autoclose);
SDL_RWops *SDL_RWFromZipFile(VFSNode *znode, VFSZipPathData *pdata);
#endif // IGUARD_rwops_rwops_zipfile_h

View file

@ -31,15 +31,13 @@ void inherit_missing_pointers(uint num, void *dest[num], void *const base[num])
}
}
bool is_main_thread(void) {
static bool initialized = false;
static SDL_threadID main_thread_id = 0;
SDL_threadID tid = SDL_ThreadID();
SDL_threadID main_thread_id = 0;
if(!initialized) {
main_thread_id = tid;
bool is_main_thread(void) {
if(main_thread_id == 0) {
return true;
}
SDL_threadID tid = SDL_ThreadID();
return main_thread_id == tid;
}

View file

@ -11,10 +11,14 @@
#include "taisei.h"
#include <SDL.h>
void* memdup(const void *src, size_t size);
void inherit_missing_pointers(uint num, void *dest[num], void *const base[num]);
bool is_main_thread(void);
extern SDL_threadID main_thread_id;
#define ARRAY_SIZE(arr) (sizeof(arr)/sizeof(*(arr)))
#endif // IGUARD_util_crap_h

View file

@ -383,8 +383,6 @@ bool pixmap_load_stream(SDL_RWops *stream, Pixmap *dst) {
}
bool pixmap_load_file(const char *path, Pixmap *dst) {
// TODO: Make this work without having to read the whole file into memory
// (that is what the VFS_MODE_SEEKABLE bit currently does with zip archives).
SDL_RWops *stream = vfs_open(path, VFS_MODE_READ | VFS_MODE_SEEKABLE);
if(!stream) {

View file

@ -11,8 +11,6 @@
#include "zipfile.h"
#include "zipfile_impl.h"
static VFSZipFileTLS* vfs_zipfile_get_tls(VFSNode *node, bool create);
#define LOG_SDL_ERROR log_debug("SDL error: %s", SDL_GetError())
static zip_int64_t vfs_zipfile_srcfunc(void *userdata, void *data, zip_uint64_t len, zip_source_cmd_t cmd) {
@ -192,7 +190,7 @@ static VFSNode* vfs_zipfile_locate(VFSNode *node, const char *path) {
}
VFSNode *n = vfs_alloc();
vfs_zippath_init(n, node, tls, idx);
vfs_zippath_init(n, node, idx);
return n;
}
@ -312,7 +310,7 @@ static void vfs_zipfile_init_pathmap(VFSNode *node) {
}
}
static VFSZipFileTLS* vfs_zipfile_get_tls(VFSNode *node, bool create) {
VFSZipFileTLS* vfs_zipfile_get_tls(VFSNode *node, bool create) {
VFSZipFileData *zdata = node->data1;
VFSZipFileTLS *tls = SDL_TLSGet(zdata->tls_id);

View file

@ -44,11 +44,13 @@ void vfs_zipfile_iter_stop(VFSNode *node, void **opaque);
typedef struct VFSZipPathData {
VFSNode *zipnode;
VFSZipFileTLS *tls;
uint64_t index;
ssize_t size;
VFSInfo info;
bool seekable;
} VFSZipPathData;
void vfs_zippath_init(VFSNode *node, VFSNode *zipnode, VFSZipFileTLS *tls, zip_int64_t idx);
void vfs_zippath_init(VFSNode *node, VFSNode *zipnode, zip_int64_t idx);
VFSZipFileTLS* vfs_zipfile_get_tls(VFSNode *node, bool create);
#endif // IGUARD_vfs_zipfile_impl_h

View file

@ -13,9 +13,11 @@
#include "syspath.h"
#include "rwops/all.h"
#define ZTLS(pdata) vfs_zipfile_get_tls((pdata)->zipnode, true)
static const char* vfs_zippath_name(VFSNode *node) {
VFSZipPathData *zdata = node->data1;
return zip_get_name(zdata->tls->zip, zdata->index, 0);
return zip_get_name(ZTLS(zdata)->zip, zdata->index, 0);
}
static void vfs_zippath_free(VFSNode *node) {
@ -65,14 +67,14 @@ static const char* vfs_zippath_iter(VFSNode *node, void **opaque) {
if(!idata) {
idata = calloc(1, sizeof(VFSZipFileIterData));
idata->num = zip_get_num_entries(zdata->tls->zip, 0);
idata->num = zip_get_num_entries(ZTLS(zdata)->zip, 0);
idata->idx = zdata->index;
idata->prefix = vfs_zippath_name(node);
idata->prefix_len = strlen(idata->prefix);
*opaque = idata;
}
return vfs_zipfile_iter_shared(node, zdata->zipnode->data1, idata, zdata->tls);
return vfs_zipfile_iter_shared(node, zdata->zipnode->data1, idata, ZTLS(zdata));
}
#define vfs_zippath_iter_stop vfs_zipfile_iter_stop
@ -84,23 +86,14 @@ static SDL_RWops* vfs_zippath_open(VFSNode *node, VFSOpenMode mode) {
}
VFSZipPathData *zdata = node->data1;
zip_file_t *zipfile = zip_fopen_index(zdata->tls->zip, zdata->index, 0);
if(!zipfile) {
vfs_set_error("ZIP error: %s", zip_error_strerror(&zdata->tls->error));
return NULL;
if(mode & VFS_MODE_SEEKABLE && !zdata->seekable) {
char *repr = vfs_node_repr(node, true);
log_warn("Opening compressed file '%s' in seekable mode, this is suboptimal. Consider storing this file without compression", repr);
free(repr);
}
SDL_RWops *ziprw = SDL_RWFromZipFile(zipfile, true);
assert(ziprw != NULL);
if(!(mode & VFS_MODE_SEEKABLE)) {
return ziprw;
}
SDL_RWops *bufrw = SDL_RWCopyToBuffer(ziprw);
SDL_RWclose(ziprw);
return bufrw;
return SDL_RWFromZipFile(node, zdata);
}
static VFSNodeFuncs vfs_funcs_zippath = {
@ -116,11 +109,11 @@ static VFSNodeFuncs vfs_funcs_zippath = {
.open = vfs_zippath_open,
};
void vfs_zippath_init(VFSNode *node, VFSNode *zipnode, VFSZipFileTLS *tls, zip_int64_t idx) {
void vfs_zippath_init(VFSNode *node, VFSNode *zipnode, zip_int64_t idx) {
VFSZipPathData *zdata = calloc(1, sizeof(VFSZipPathData));
zdata->zipnode = zipnode;
zdata->tls = tls;
zdata->index = idx;
zdata->size = -1;
node->data1 = zdata;
zdata->info.exists = true;
@ -130,6 +123,20 @@ void vfs_zippath_init(VFSNode *node, VFSNode *zipnode, VFSZipFileTLS *tls, zip_i
zdata->info.is_dir = true;
}
zip_stat_t zstat;
if(zip_stat_index(ZTLS(zdata)->zip, zdata->index, 0, &zstat) < 0) {
log_warn("zip_stat_index(%"PRIi64") failed: %s", idx, zip_error_strerror(&ZTLS(zdata)->error));
} else {
if(zstat.valid & ZIP_STAT_SIZE) {
zdata->size = zstat.size;
}
if(zstat.valid & ZIP_STAT_COMP_METHOD) {
zdata->seekable = (zstat.comp_method == ZIP_CM_STORE);
}
}
node->funcs = &vfs_funcs_zippath;
const char *path = vfs_zippath_name(node);