Add a basic shader cache; move some files around

The cache is currently only used to speed up SPIR-V transpilation.

README.rst

@@ -16,23 +16,30 @@ Installation
 Dependencies
 ^^^^^^^^^^^^
 
--  SDL2 >= 2.0.5, SDL2_mixer
--  zlib
--  libzip >= 1.0
+-  OpenGL >= 3.3 or OpenGL ES >= 3.0
+-  SDL2 >= 2.0.5
+-  SDL2_mixer
+-  freetype2
 -  libpng >= 1.5.0
 -  libwebpdecoder >= 0.5 or libwebp >= 0.5
--  freetype2
--  OpenGL >= 3.3 or OpenGL ES >= 3.0
--  libshaderc (optional, for OpenGL ES backends)
--  crossc >= 1.5.0 (optional, for OpenGL ES backends)
+-  libzip >= 1.0
+-  zlib
+
+Optional:
+
+-  OpenSSL (for a better SHA-256 implementation; used in shader cache)
+-  crossc >= 1.5.0 (for OpenGL ES backends)
+-  libshaderc (for OpenGL ES backends)
 
 Build-only dependencies
 ^^^^^^^^^^^^^^^^^^^^^^^
 
--  meson >= 0.45.0 (build system)
 -  Python >= 3.4
--  pkg-config
--  docutils (optional, for documentation)
+-  meson >= 0.45.0 (build system)
+
+Optional:
+
+-  docutils (for documentation)
 
 To build and install Taisei just follow these steps.
 

meson.build

@@ -64,6 +64,7 @@ taisei_c_args = cc.get_supported_arguments(
 
 static = get_option('static')
 
+dep_crossc      = dependency('crossc',         version : '>=1.5.0', required : false, static : static, fallback : ['crossc', 'crossc_dep'])
 dep_freetype    = dependency('freetype2',                           required : true,  static : static)
 dep_png         = dependency('libpng',         version : '>=1.5',   required : true,  static : static)
 dep_sdl2        = dependency('sdl2',           version : '>=2.0.5', required : true,  static : static)
@@ -72,8 +73,17 @@ dep_webp        = dependency('libwebp',        version : '>=0.5',   required : f
 dep_webpdecoder = dependency('libwebpdecoder', version : '>=0.5',   required : false, static : static)
 dep_zip         = dependency('libzip',         version : '>=1.0',   required : false, static : static)
 dep_zlib        = dependency('zlib',                                required : true,  static : static)
+dep_crypto      = dependency('libcrypto',                           required : false, static : static)
 
-dep_m           = cc.find_library('m',                              required : false)
+# Thanks, google.
+# https://github.com/google/shaderc/issues/392
+if static
+    dep_shaderc = cc.find_library('shaderc_combined', required : false)
+else
+    dep_shaderc = cc.find_library('shaderc_shared', required : false)
+endif
+
+dep_m           = cc.find_library('m', required : false)
 
 dep_cglm        = subproject('cglm').get_variable('cglm_dep')
 dep_glad        = subproject('glad').get_variable('glad_dep')

meson_options.txt

@@ -162,3 +162,10 @@ option(
     value : true,
     description : 'Pre-allocate memory for game objects (disable for debugging only)'
 )
+
+option(
+    'use_libcrypto',
+    type : 'combo',
+    choices : ['auto', 'true', 'false'],
+    description : 'Use libcrypto from OpenSSL for better SHA implementations'
+)

src/meson.build

@@ -141,6 +141,11 @@ taisei_src += [
     vfs_src,
 ]
 
+taisei_deps += [
+    renderer_deps,
+    util_deps,
+]
+
 if taisei_deps.contains(dep_sdl2_mixer)
     taisei_src += files(
         'audio_mixer.c',
@@ -151,8 +156,6 @@ else
     )
 endif
 
-taisei_deps += renderer_deps
-
 if macos_app_bundle
     taisei_exe_name = 'Taisei'
 else

src/renderer/api.h

@@ -14,7 +14,7 @@
 #include "util.h"
 #include "util/pixmap.h"
 #include "color.h"
-#include "common/shader.h"
+#include "common/shaderlib/shaderlib.h"
 #include "resource/resource.h"
 
 typedef struct Texture Texture;

src/renderer/common/meson.build

@@ -3,9 +3,11 @@ r_common_src = files(
     'backend.c',
     'matstack.c',
     'models.c',
-    'shader_glsl.c',
     'sprite_batch.c',
     'state.c',
 )
 
-r_common_libdeps = []
+subdir('shaderlib')
+
+r_common_src += r_shaderlib_src
+r_common_libdeps = r_shaderlib_libdeps

src/renderer/common/shaderlib/cache.c

@@ -0,0 +1,203 @@
+/*
+ * This software is licensed under the terms of the MIT-License
+ * See COPYING for further information.
+ * ---
+ * Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
+ * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
+ */
+
+#include "taisei.h"
+
+#include "shaderlib.h"
+
+#include "util.h"
+#include "util/sha256.h"
+#include "rwops/all.h"
+
+#define CACHE_VERSION 1
+#define CRC_INIT 0
+
+static uint8_t* shader_cache_construct_entry(const ShaderSource *src, size_t *out_size) {
+	uint8_t *buf, *result = NULL;
+	uint32_t crc = CRC_INIT;
+
+	SDL_RWops *abuf = SDL_RWAutoBuffer((void**)&buf, BUFSIZ);
+	SDL_RWops *crcbuf = SDL_RWWrapCRC32(abuf, &crc, false);
+	SDL_RWops *dest = crcbuf;
+
+	SDL_WriteU8(dest, CACHE_VERSION);
+	SDL_WriteU8(dest, src->stage);
+	SDL_WriteU8(dest, src->lang.lang);
+
+	switch(src->lang.lang) {
+		case SHLANG_GLSL: {
+			SDL_WriteLE16(dest, src->lang.glsl.version.version);
+			SDL_WriteU8(dest, src->lang.glsl.version.profile);
+			break;
+		}
+
+		case SHLANG_SPIRV: {
+			SDL_WriteU8(dest, src->lang.spirv.target);
+			break;
+		}
+
+		default: {
+			log_warn("Unhandled shading language id=%u", src->lang.lang);
+			goto fail;
+		}
+	}
+
+	SDL_WriteLE64(dest, src->content_size);
+	SDL_RWwrite(dest, src->content, src->content_size, 1);
+
+	dest = abuf;
+	SDL_RWclose(crcbuf);
+	crcbuf = NULL;
+
+	SDL_WriteLE32(dest, crc);
+
+	*out_size = SDL_RWtell(dest);
+	result = memdup(buf, *out_size);
+
+fail:
+	if(crcbuf != NULL) {
+		SDL_RWclose(crcbuf);
+	}
+
+	SDL_RWclose(abuf);
+	return result;
+}
+
+static bool shader_cache_load_entry(SDL_RWops *stream, ShaderSource *out_src) {
+	uint32_t crc = CRC_INIT;
+	SDL_RWops *s = SDL_RWWrapCRC32(stream, &crc, false);
+
+	out_src->content = NULL;
+
+	if(SDL_ReadU8(s) != CACHE_VERSION) {
+		log_warn("Version mismatch");
+		goto fail;
+	}
+
+	out_src->stage = SDL_ReadU8(s);
+
+	memset(&out_src->lang, 0, sizeof(out_src->lang));
+	out_src->lang.lang = SDL_ReadU8(s);
+
+	switch(out_src->lang.lang) {
+		case SHLANG_GLSL: {
+			out_src->lang.glsl.version.version = SDL_ReadLE16(s);
+			out_src->lang.glsl.version.profile = SDL_ReadU8(s);
+			break;
+		}
+
+		case SHLANG_SPIRV: {
+			out_src->lang.spirv.target = SDL_ReadU8(s);
+			break;
+		}
+
+		default: {
+			log_warn("Unhandled shading language id=%u", out_src->lang.lang);
+			goto fail;
+		}
+	}
+
+	out_src->content_size = SDL_ReadLE64(s);
+	out_src->content = calloc(1, out_src->content_size);
+
+	if(SDL_RWread(s, out_src->content, out_src->content_size, 1) != 1) {
+		log_warn("Read error");
+		goto fail;
+	}
+
+	uint32_t file_crc = SDL_ReadLE32(stream);
+
+	if(crc != file_crc) {
+		log_warn("CRC mismatch (%08x != %08x), cache entry is corrupted", crc, file_crc);
+		goto fail;
+	}
+
+	SDL_RWclose(s);
+	return true;
+
+fail:
+	free(out_src->content);
+	out_src->content = NULL;
+	SDL_RWclose(s);
+	return false;
+}
+
+bool shader_cache_get(const char *hash, const char *key, ShaderSource *entry) {
+	char path[256];
+	snprintf(path, sizeof(path), "cache/shaders/%s/%s", hash, key);
+
+	SDL_RWops *stream = vfs_open(path, VFS_MODE_READ);
+
+	if(stream == NULL) {
+		return false;
+	}
+
+	stream = SDL_RWWrapZReader(stream, BUFSIZ, true);
+	bool result = shader_cache_load_entry(stream, entry);
+	SDL_RWclose(stream);
+
+	return result;
+}
+
+static bool shader_cache_set_raw(const char *hash, const char *key, uint8_t *entry, size_t entry_size) {
+	char path[256];
+
+	vfs_mkdir("cache/shaders");
+	snprintf(path, sizeof(path), "cache/shaders/%s", hash);
+	vfs_mkdir(path);
+	snprintf(path, sizeof(path), "cache/shaders/%s/%s", hash, key);
+
+	SDL_RWops *out = vfs_open(path, VFS_MODE_WRITE);
+
+	if(out == NULL) {
+		log_warn("VFS error: %s", vfs_get_error());
+		return false;
+	}
+
+	out = SDL_RWWrapZWriter(out, BUFSIZ, true);
+	assert(out != NULL);
+
+	SDL_RWwrite(out, entry, entry_size, 1);
+	SDL_RWclose(out);
+
+	return true;
+}
+
+bool shader_cache_set(const char *hash, const char *key, const ShaderSource *src) {
+	size_t entry_size;
+	uint8_t *entry = shader_cache_construct_entry(src, &entry_size);
+
+	if(entry != NULL) {
+		shader_cache_set_raw(hash, key, entry, entry_size);
+		free(entry);
+		return true;
+	}
+
+	return false;
+}
+
+bool shader_cache_hash(const ShaderSource *src, char *out_buf, size_t bufsize) {
+	assert(bufsize >= SHADER_CACHE_HASH_BUFSIZE);
+
+	size_t entry_size;
+	uint8_t *entry = shader_cache_construct_entry(src, &entry_size);
+
+	if(entry == NULL) {
+		return false;
+	}
+
+	const size_t sha_size = SHA256_BLOCK_SIZE*2;
+
+	sha256_hexdigest(entry, entry_size, out_buf, bufsize);
+	snprintf(out_buf + sha_size, bufsize - sha_size, "-%08zu", entry_size);
+
+	// shader_cache_set_raw(out_buf, "orig", entry, entry_size);
+
+	free(entry);
+	return true;
+}

src/renderer/common/shaderlib/cache.h

@@ -0,0 +1,26 @@
+/*
+ * This software is licensed under the terms of the MIT-License
+ * See COPYING for further information.
+ * ---
+ * Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
+ * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
+ */
+
+#ifndef IGUARD_renderer_common_shaderlib_cache_h
+#define IGUARD_renderer_common_shaderlib_cache_h
+
+#include "taisei.h"
+
+#include "defs.h"
+
+// sha256 hexdigest  : 64 bytes
+// separator         : 1 byte
+// 64-bit size (hex) : 8 bytes
+// null terminator   : 1 byte
+#define SHADER_CACHE_HASH_BUFSIZE 74
+
+bool shader_cache_hash(const ShaderSource *src, char *out_buf, size_t bufsize) attr_nonnull(1, 2) attr_nodiscard;
+bool shader_cache_get(const char *hash, const char *key, ShaderSource *entry) attr_nonnull(1, 2, 3) attr_nodiscard;
+bool shader_cache_set(const char *hash, const char *key, const ShaderSource *src) attr_nonnull(1, 2, 3);
+
+#endif // IGUARD_renderer_common_shaderlib_cache_h

src/renderer/common/shaderlib/defs.h

@@ -6,8 +6,8 @@
  * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
  */
 
-#ifndef IGUARD_renderer_common_shader_defs_h
-#define IGUARD_renderer_common_shader_defs_h
+#ifndef IGUARD_renderer_common_shaderlib_defs_h
+#define IGUARD_renderer_common_shaderlib_defs_h
 
 #include "taisei.h"
 
@@ -26,4 +26,4 @@ typedef enum ShaderLanguage {
 typedef struct ShaderSource ShaderSource;
 typedef struct ShaderLangInfo ShaderLangInfo;
 
-#endif // IGUARD_renderer_common_shader_defs_h
+#endif // IGUARD_renderer_common_shaderlib_defs_h

src/renderer/common/shaderlib/lang_glsl.c

@@ -8,7 +8,7 @@
 
 #include "taisei.h"
 
-#include "shader.h"
+#include "shaderlib.h"
 #include "rwops/rwops_autobuf.h"
 #include "vfs/pathutil.h"
 #include "vfs/public.h"

src/renderer/common/shaderlib/lang_glsl.h

@@ -6,12 +6,12 @@
  * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
  */
 
-#ifndef IGUARD_renderer_common_shader_glsl_h
-#define IGUARD_renderer_common_shader_glsl_h
+#ifndef IGUARD_renderer_common_shaderlib_lang_glsl_h
+#define IGUARD_renderer_common_shaderlib_lang_glsl_h
 
 #include "taisei.h"
 
-#include "shader_defs.h"
+#include "defs.h"
 
 typedef enum GLSLProfile {
 	GLSL_PROFILE_NONE,
@@ -41,4 +41,4 @@ bool glsl_load_source(const char *path, ShaderSource *out, const GLSLSourceOptio
 char* glsl_parse_version(const char *str, GLSLVersion *out_version);
 int glsl_format_version(char *buf, size_t bufsize, GLSLVersion version);
 
-#endif // IGUARD_renderer_common_shader_glsl_h
+#endif // IGUARD_renderer_common_shaderlib_lang_glsl_h

src/renderer/common/shaderlib/lang_spirv.c

@@ -8,11 +8,13 @@
 
 #include "taisei.h"
 
-#include "../common/shader.h"
+#include "shaderlib.h"
+#include "lang_spirv_private.h"
 
 #include "util.h"
-#include "shaderc/shaderc.h"
-#include "crossc.h"
+
+#include <shaderc/shaderc.h>
+#include <crossc.h>
 
 static shaderc_compiler_t spirv_compiler;
 
@@ -77,7 +79,7 @@ void spirv_shutdown_compiler(void) {
 	}
 }
 
-bool spirv_compile(const ShaderSource *in, ShaderSource *out, const SPIRVCompileOptions *options) {
+bool _spirv_compile(const ShaderSource *in, ShaderSource *out, const SPIRVCompileOptions *options) {
 	if(in->lang.lang != SHLANG_GLSL) {
 		log_warn("Unsupported source language");
 		return false;
@@ -184,7 +186,7 @@ bool spirv_compile(const ShaderSource *in, ShaderSource *out, const SPIRVCompile
 	return true;
 }
 
-bool spirv_decompile(const ShaderSource *in, ShaderSource *out, const SPIRVDecompileOptions *options) {
+bool _spirv_decompile(const ShaderSource *in, ShaderSource *out, const SPIRVDecompileOptions *options) {
 	if(in->lang.lang != SHLANG_SPIRV) {
 		log_warn("Source is not a SPIR-V binary");
 		return false;
@@ -236,62 +238,3 @@ crossc_error:
 	crossc_destroy(cc);
 	return false;
 }
-
-static bool lang_supports_uniform_locations(const ShaderLangInfo *lang) {
-	if(lang->lang != SHLANG_GLSL) {
-		return false; // FIXME?
-	}
-
-	if(lang->glsl.version.profile == GLSL_PROFILE_ES) {
-		return lang->glsl.version.version >= 310;
-	} else {
-		return lang->glsl.version.version >= 430;
-	}
-}
-
-bool spirv_transpile(const ShaderSource *in, ShaderSource *out, const SPIRVTranspileOptions *options) {
-	ShaderSource spirv = { 0 };
-	bool result;
-
-	ShaderSource _in = *in;
-	in = &_in;
-
-	if(
-		!lang_supports_uniform_locations(&_in.lang) &&
-		lang_supports_uniform_locations(options->lang) &&
-		_in.lang.lang == SHLANG_GLSL
-	) {
-		// HACK: This is annoying... shaderc/glslang does not support GL_ARB_explicit_uniform_location
-		// for some reason. Until there's a better solution, we'll try to compile the shader using a
-		// higher version.
-
-		if(_in.lang.glsl.version.profile == GLSL_PROFILE_ES) {
-			_in.lang.glsl.version.version = 310;
-		} else {
-			_in.lang.glsl.version.version = 430;
-		}
-	}
-
-	result = spirv_compile(in, &spirv, &(SPIRVCompileOptions) {
-		.target = SPIRV_TARGET_OPENGL_450, // TODO: specify this in the shader
-		.optimization_level = options->optimization_level,
-		.filename = options->filename,
-
-		// Preserve names of declarations.
-		// This can be vital for shader interface matching.
-		.debug_info = true,
-	});
-
-	if(result) {
-		result = spirv_decompile(&spirv, out, &(SPIRVDecompileOptions) {
-			.lang = options->lang,
-		});
-
-		if(result) {
-			log_debug("%s: translated code:\n%s", options->filename, out->content);
-		}
-	}
-
-	free(spirv.content);
-	return result;
-}

src/renderer/common/shaderlib/lang_spirv.h

@@ -6,12 +6,12 @@
  * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
  */
 
-#ifndef IGUARD_renderer_common_shader_spirv_h
-#define IGUARD_renderer_common_shader_spirv_h
+#ifndef IGUARD_renderer_common_shaderlib_lang_spirv_h
+#define IGUARD_renderer_common_shaderlib_lang_spirv_h
 
 #include "taisei.h"
 
-#include "shader_defs.h"
+#include "defs.h"
 
 typedef enum SPIRVTarget {
 	SPIRV_TARGET_OPENGL_450,
@@ -44,8 +44,9 @@ typedef struct SPIRVTranspileOptions {
 
 void spirv_init_compiler(void);
 void spirv_shutdown_compiler(void);
+
+bool spirv_transpile(const ShaderSource *in, ShaderSource *out, const SPIRVTranspileOptions *options) attr_nonnull(1, 2, 3);
 bool spirv_compile(const ShaderSource *in, ShaderSource *out, const SPIRVCompileOptions *options) attr_nonnull(1, 2, 3);
 bool spirv_decompile(const ShaderSource *in, ShaderSource *out, const SPIRVDecompileOptions *options) attr_nonnull(1, 2, 3);
-bool spirv_transpile(const ShaderSource *in, ShaderSource *out, const SPIRVTranspileOptions *options);
 
-#endif // IGUARD_renderer_common_shader_spirv_h
+#endif // IGUARD_renderer_common_shaderlib_lang_spirv_h

src/renderer/common/shaderlib/lang_spirv_aux.c

@@ -0,0 +1,161 @@
+/*
+ * This software is licensed under the terms of the MIT-License
+ * See COPYING for further information.
+ * ---
+ * Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
+ * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
+ */
+
+#include "taisei.h"
+
+#include "shaderlib.h"
+#include "lang_spirv_private.h"
+#include "util.h"
+
+static bool shader_cache_entry_name(const ShaderLangInfo *lang, ShaderStage stage, SPIRVOptimizationLevel optlvl, char *buf, size_t bufsize) {
+	// TODO: somehow get rid of this ugly optlvl parameter and generalize external metadata?
+
+	switch(lang->lang) {
+		case SHLANG_GLSL: {
+			snprintf(buf, bufsize, "glsl_%u_%u_%u", stage, lang->glsl.version.version, lang->glsl.version.profile);
+			return true;
+		}
+
+		case SHLANG_SPIRV: {
+			snprintf(buf, bufsize, "spirv_%u_%u_%u", stage, lang->spirv.target, optlvl);
+			return true;
+		}
+
+		default: {
+			log_warn("Unhandled shading language id=%u", lang->lang);
+			return false;
+		}
+	}
+
+	return false;
+}
+
+bool spirv_compile(const ShaderSource *in, ShaderSource *out, const SPIRVCompileOptions *options) {
+	char name[256], hash[SHADER_CACHE_HASH_BUFSIZE];
+
+	ShaderLangInfo target_lang = { 0 };
+	target_lang.lang = SHLANG_SPIRV;
+	target_lang.spirv.target = options->target;
+
+	if(!shader_cache_entry_name(&target_lang, in->stage, options->optimization_level, name, sizeof(name))) {
+		return _spirv_compile(in, out, options);
+	}
+
+	if(!shader_cache_hash(in, hash, sizeof(hash))) {
+		return _spirv_compile(in, out, options);
+	}
+
+	bool result;
+
+	if((result = shader_cache_get(hash, name, out))) {
+		if(
+			!memcmp(&target_lang, &out->lang, sizeof(ShaderLangInfo)) &&
+			out->stage == in->stage
+		) {
+			return true;
+		}
+
+		log_warn("Invalid cache entry ignored");
+	}
+
+	if((result = _spirv_compile(in, out, options))) {
+		shader_cache_set(hash, name, out);
+	}
+
+	return result;
+}
+
+bool spirv_decompile(const ShaderSource *in, ShaderSource *out, const SPIRVDecompileOptions *options) {
+	char name[256], hash[SHADER_CACHE_HASH_BUFSIZE];
+
+	if(!shader_cache_entry_name(options->lang, in->stage, 0, name, sizeof(name))) {
+		return _spirv_decompile(in, out, options);
+	}
+
+	if(!shader_cache_hash(in, hash, sizeof(hash))) {
+		return _spirv_decompile(in, out, options);
+	}
+
+	bool result;
+
+	if((result = shader_cache_get(hash, name, out))) {
+		if(
+			!memcmp(options->lang, &out->lang, sizeof(ShaderLangInfo)) &&
+			out->stage == in->stage
+		) {
+			return true;
+		}
+
+		log_warn("Invalid cache entry ignored");
+	}
+
+	if((result = _spirv_decompile(in, out, options))) {
+		shader_cache_set(hash, name, out);
+	}
+
+	return result;
+}
+
+static bool lang_supports_uniform_locations(const ShaderLangInfo *lang) {
+	if(lang->lang != SHLANG_GLSL) {
+		return false; // FIXME?
+	}
+
+	if(lang->glsl.version.profile == GLSL_PROFILE_ES) {
+		return lang->glsl.version.version >= 310;
+	} else {
+		return lang->glsl.version.version >= 430;
+	}
+}
+
+bool spirv_transpile(const ShaderSource *in, ShaderSource *out, const SPIRVTranspileOptions *options) {
+	ShaderSource spirv = { 0 };
+	bool result;
+
+	ShaderSource _in = *in;
+	in = &_in;
+
+	if(
+		!lang_supports_uniform_locations(&_in.lang) &&
+		lang_supports_uniform_locations(options->lang) &&
+		_in.lang.lang == SHLANG_GLSL
+	) {
+		// HACK: This is annoying... shaderc/glslang does not support GL_ARB_explicit_uniform_location
+		// for some reason. Until there's a better solution, we'll try to compile the shader using a
+		// higher version.
+
+		if(_in.lang.glsl.version.profile == GLSL_PROFILE_ES) {
+			_in.lang.glsl.version.version = 310;
+		} else {
+			_in.lang.glsl.version.version = 430;
+		}
+	}
+
+	result = spirv_compile(in, &spirv, &(SPIRVCompileOptions) {
+		.target = SPIRV_TARGET_OPENGL_450, // TODO: specify this in the shader
+		.optimization_level = options->optimization_level,
+		.filename = options->filename,
+
+		// Preserve names of declarations.
+		// This can be vital for shader interface matching.
+		.debug_info = true,
+	});
+
+	if(result) {
+		result = spirv_decompile(&spirv, out, &(SPIRVDecompileOptions) {
+			.lang = options->lang,
+		});
+
+		if(result) {
+			log_debug("%s: translated code:\n%s", options->filename, out->content);
+		}
+	}
+
+	free(spirv.content);
+	return result;
+}

src/renderer/common/shaderlib/lang_spirv_private.h

@@ -0,0 +1,19 @@
+/*
+ * This software is licensed under the terms of the MIT-License
+ * See COPYING for further information.
+ * ---
+ * Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
+ * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
+ */
+
+#ifndef IGUARD_renderer_common_shaderlib_lang_spirv_private_h
+#define IGUARD_renderer_common_shaderlib_lang_spirv_private_h
+
+#include "taisei.h"
+
+#include "lang_spirv.h"
+
+bool _spirv_compile(const ShaderSource *in, ShaderSource *out, const SPIRVCompileOptions *options) attr_nonnull(1, 2, 3);
+bool _spirv_decompile(const ShaderSource *in, ShaderSource *out, const SPIRVDecompileOptions *options) attr_nonnull(1, 2, 3);
+
+#endif // IGUARD_renderer_common_shaderlib_lang_spirv_private_h

src/renderer/common/shaderlib/lang_spirv_stub.c

@@ -8,23 +8,19 @@
 
 #include "taisei.h"
 
-#include "../common/shader_spirv.h"
+#include "shaderlib.h"
+#include "lang_spirv_private.h"
 #include "util.h"
 
 void spirv_init_compiler(void) { }
 void spirv_shutdown_compiler(void) { }
 
-bool spirv_compile(const ShaderSource *in, ShaderSource *out, const SPIRVCompileOptions *options) {
+bool _spirv_compile(const ShaderSource *in, ShaderSource *out, const SPIRVCompileOptions *options) {
 	log_warn("Compiled without SPIR-V support");
 	return false;
 }
 
-bool spirv_decompile(const ShaderSource *in, ShaderSource *out, const SPIRVDecompileOptions *options) {
-	log_warn("Compiled without SPIR-V support");
-	return false;
-}
-
-bool spirv_transpile(const ShaderSource *in, ShaderSource *out, const SPIRVTranspileOptions *options) {
+bool _spirv_decompile(const ShaderSource *in, ShaderSource *out, const SPIRVDecompileOptions *options) {
 	log_warn("Compiled without SPIR-V support");
 	return false;
 }

src/renderer/common/shaderlib/meson.build

@@ -0,0 +1,34 @@
+
+r_shaderlib_src = files(
+    'cache.c',
+    'lang_glsl.c',
+    'lang_spirv_aux.c',
+)
+
+r_shaderlib_libdeps = []
+
+if get_option('shader_transpiler')
+    assert(dep_shaderc.found() and dep_crossc.found(),
+        'shaderc and crossc are required for shader translation. Install them or disable shader_transpiler.'
+    )
+
+    r_shaderlib_src += files(
+        'lang_spirv.c'
+    )
+
+    r_shaderlib_libdeps += [dep_shaderc, dep_crossc]
+
+    if host_machine.system() == 'windows' and get_option('b_lto') and get_option('static')
+        error(
+    '''
+
+    -!-  LTO is known to break glslang on Windows.
+    -!-  Please disable it with `meson configure -Db_lto=false`.
+    -!-  If you *really* want to use LTO, force it via compiler args or patch this check out.
+    ''')
+    endif
+else
+    r_shaderlib_src += files(
+        'lang_spirv_stub.c'
+    )
+endif

src/renderer/common/shaderlib/shaderlib.h

@@ -6,14 +6,16 @@
  * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
  */
 
-#ifndef IGUARD_renderer_common_shader_h
-#define IGUARD_renderer_common_shader_h
+#ifndef IGUARD_renderer_common_shaderlib_shaderlib_h
+#define IGUARD_renderer_common_shaderlib_shaderlib_h
 
 #include "taisei.h"
 
-#include "shader_defs.h"
-#include "shader_glsl.h"
-#include "shader_spirv.h"
+#include "defs.h"
+#include "cache.h"
+
+#include "lang_glsl.h"
+#include "lang_spirv.h"
 
 struct ShaderLangInfo {
 	ShaderLanguage lang;
@@ -36,4 +38,4 @@ struct ShaderSource {
 	ShaderStage stage;
 };
 
-#endif // IGUARD_renderer_common_shader_h
+#endif // IGUARD_renderer_common_shaderlib_shaderlib_h

src/renderer/glcommon/shaders.c

@@ -9,7 +9,6 @@
 #include "taisei.h"
 
 #include "shaders.h"
-#include "../common/shader_glsl.h"
 #include "util.h"
 #include "opengl.h"
 #include "rwops/rwops_autobuf.h"

src/renderer/glcommon/shaders.h

@@ -11,7 +11,7 @@
 
 #include "taisei.h"
 
-#include "../common/shader.h"
+#include "../common/shaderlib/shaderlib.h"
 
 extern ShaderLangInfo *glcommon_shader_lang_table;
 

src/renderer/meson.build

@@ -31,14 +31,6 @@ included_deps = []
 needed_deps = ['common']
 r_macro = []
 
-if get_option('shader_transpiler')
-    subdir('spirv_tools')
-    needed_deps += ['spirv_tools']
-else
-    subdir('spirv_tools_stub')
-    needed_deps += ['spirv_tools_stub']
-endif
-
 foreach m : modules
     if get_option('r_@0@'.format(m))
         renderer_src += get_variable('r_@0@_src'.format(m))

src/renderer/spirv_tools/meson.build

@@ -1,31 +0,0 @@
-
-# Thanks, google.
-# https://github.com/google/shaderc/issues/392
-if get_option('static')
-    dep_shaderc = cc.find_library('shaderc_combined', required: true)
-else
-    dep_shaderc = cc.find_library('shaderc_shared', required: true)
-endif
-
-dep_crossc = dependency('crossc',
-    required : true,
-    static : get_option('static'),
-    version : '>=1.5.0',
-    fallback : ['crossc', 'crossc_dep']
-)
-
-r_spirv_tools_src = files(
-    'shader_spirv.c'
-)
-
-r_spirv_tools_libdeps = [dep_shaderc, dep_crossc]
-
-if host_machine.system() == 'windows' and get_option('b_lto') and get_option('static')
-    error(
-'''
-
- -!-  LTO is known to break glslang on Windows.
- -!-  Please disable it with `meson configure -Db_lto=false`.
- -!-  If you *really* want to use LTO, force it via compiler args or patch this check out.
-''')
-endif

src/renderer/spirv_tools_stub/meson.build

@@ -1,6 +0,0 @@
-
-r_spirv_tools_stub_src = files(
-    'shader_spirv.c'
-)
-
-r_spirv_tools_stub_libdeps = []

src/rwops/all.h

@@ -11,11 +11,11 @@
 
 #include "taisei.h"
 
-#include "build_config.h"
-#include "rwops_zlib.h"
-#include "rwops_segment.h"
 #include "rwops_autobuf.h"
+#include "rwops_crc32.h"
 #include "rwops_pipe.h"
+#include "rwops_segment.h"
+#include "rwops_zlib.h"
 
 #ifdef TAISEI_BUILDCONF_USE_ZIP
 #include "rwops_zipfile.h"

src/rwops/meson.build

@@ -1,6 +1,7 @@
 
 rwops_src = files(
     'rwops_autobuf.c',
+    'rwops_crc32.c',
     'rwops_dummy.c',
     'rwops_segment.c',
     'rwops_zlib.c',

src/rwops/rwops_crc32.c

@@ -0,0 +1,78 @@
+/*
+ * This software is licensed under the terms of the MIT-License
+ * See COPYING for further information.
+ * ---
+ * Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
+ * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
+ */
+
+#include "taisei.h"
+
+#include "rwops_crc32.h"
+#include "util.h"
+
+#include <zlib.h>
+
+struct crc32_data {
+	SDL_RWops *src;
+	uint32_t *crc32_ptr;
+	bool autoclose;
+};
+
+#define DATA(rw) ((struct crc32_data*)((rw)->hidden.unknown.data1))
+
+static int rwcrc32_close(SDL_RWops *rw) {
+	if(DATA(rw)->autoclose) {
+		SDL_RWclose(DATA(rw)->src);
+	}
+
+	free(DATA(rw));
+	SDL_FreeRW(rw);
+	return 0;
+}
+
+static int64_t rwcrc32_seek(SDL_RWops *rw, int64_t offset, int whence) {
+	SDL_SetError("Stream is not seekable");
+	return -1;
+}
+
+static int64_t rwcrc32_size(SDL_RWops *rw) {
+	return SDL_RWsize(DATA(rw)->src);
+}
+
+static void rwcrc32_update_crc(SDL_RWops *rw, const void *data, size_t size, size_t maxnum) {
+	assert(size <= UINT32_MAX);
+	assert(maxnum <= UINT32_MAX);
+	assert(size * maxnum <= UINT32_MAX);
+	*DATA(rw)->crc32_ptr = crc32(*DATA(rw)->crc32_ptr, data, size * maxnum);
+}
+
+static size_t rwcrc32_read(SDL_RWops *rw, void *ptr, size_t size, size_t maxnum) {
+	size_t result = SDL_RWread(DATA(rw)->src, ptr, size, maxnum);
+	rwcrc32_update_crc(rw, ptr, size, maxnum);
+	return result;
+}
+
+static size_t rwcrc32_write(SDL_RWops *rw, const void *ptr, size_t size, size_t maxnum) {
+	rwcrc32_update_crc(rw, ptr, size, maxnum);
+	return SDL_RWwrite(DATA(rw)->src, ptr, size, maxnum);
+}
+
+SDL_RWops* SDL_RWWrapCRC32(SDL_RWops *src, uint32_t *crc32_ptr, bool autoclose) {
+	SDL_RWops *rw = SDL_AllocRW();
+	memset(rw, 0, sizeof(SDL_RWops));
+
+	rw->type = SDL_RWOPS_UNKNOWN;
+	rw->hidden.unknown.data1 = calloc(1, sizeof(struct crc32_data));
+	DATA(rw)->src = src;
+	DATA(rw)->crc32_ptr = crc32_ptr;
+	DATA(rw)->autoclose = autoclose;
+
+	rw->size = rwcrc32_size;
+	rw->seek = rwcrc32_seek;
+	rw->close = rwcrc32_close;
+	rw->read = rwcrc32_read;
+	rw->write = rwcrc32_write;
+
+	return rw;
+}

src/rwops/rwops_crc32.h

@@ -0,0 +1,18 @@
+/*
+ * This software is licensed under the terms of the MIT-License
+ * See COPYING for further information.
+ * ---
+ * Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
+ * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
+ */
+
+#ifndef IGUARD_rwops_rwops_crc32_h
+#define IGUARD_rwops_rwops_crc32_h
+
+#include "taisei.h"
+
+#include <SDL.h>
+
+SDL_RWops* SDL_RWWrapCRC32(SDL_RWops *src, uint32_t *crc32_ptr, bool autoclose);
+
+#endif // IGUARD_rwops_rwops_crc32_h

src/rwops/rwops_zlib.c

@@ -265,7 +265,7 @@ static size_t inflate_read(SDL_RWops *rw, void *ptr, size_t size, size_t maxnum)
 	}
 
 	z->pos += (totalsize - z->stream->avail_out);
-	return (totalsize - z->stream->avail_out) / maxnum;
+	return (totalsize - z->stream->avail_out) / size;
 }
 
 static size_t inflate_write(SDL_RWops *rw, const void *ptr, size_t size, size_t maxnum) {

src/util/meson.build

@@ -1,4 +1,6 @@
 
+util_deps = []
+
 util_src = files(
     'assert.c',
     'crap.c',
@@ -33,6 +35,26 @@ else
     config.set('TAISEI_BUILDCONF_USE_DESIGNATED_INIT', false)
 endif
 
+use_libcrypto = get_option('use_libcrypto')
+
+if use_libcrypto == 'auto'
+    use_libcrypto = dep_crypto.found()
+else
+    use_libcrypto = (use_libcrypto == 'true')
+endif
+
+if use_libcrypto
+    assert(dep_crypto.found(), 'use_libcrypto forced but libcrypto not found. Install OpenSSL or disable use_libcrypto.')
+    util_src += files('sha256_openssl.c')
+    util_deps += [dep_crypto]
+else
+    assert(host_machine.endian() == 'little',
+        'The built-in SHA-256 implementation only supports little-endian CPUs. Enable use_libcrypto to use the OpenSSL version, or send a patch with a better implementation :)'
+    )
+
+    util_src += files('sha256_le.c')
+endif
+
 if host_machine.system() == 'windows'
     # NOTE: Even if we ever build this with something like Midipix, we'd
     # probably still want to use the winapi implementation of this here.

src/util/sha256.h

@@ -0,0 +1,26 @@
+/*
+ * This software is licensed under the terms of the MIT-License
+ * See COPYING for further information.
+ * ---
+ * Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
+ * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
+ */
+
+#ifndef IGUARD_util_sha256_h
+#define IGUARD_util_sha256_h
+
+#include "taisei.h"
+
+#define SHA256_BLOCK_SIZE 32
+
+typedef struct SHA256State SHA256State;
+
+SHA256State* sha256_new(void) attr_nodiscard;
+void sha256_update(SHA256State *state, const uint8_t *data, size_t len) attr_nonnull(1, 2);
+void sha256_final(SHA256State *state, uint8_t hash[SHA256_BLOCK_SIZE], size_t hashlen) attr_nonnull(1, 2);
+void sha256_free(SHA256State *state);
+
+void sha256_digest(const uint8_t *data, size_t len, uint8_t hash[SHA256_BLOCK_SIZE], size_t hashlen);
+void sha256_hexdigest(const uint8_t *data, size_t len, char hash[SHA256_BLOCK_SIZE*2+1], size_t hashlen);
+
+#endif // IGUARD_util_sha256_h

src/util/sha256_le.c

@@ -0,0 +1,217 @@
+/*
+ * This software is licensed under the terms of the MIT-License
+ * See COPYING for further information.
+ * ---
+ * Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
+ * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
+ */
+
+#include "taisei.h"
+
+#include "sha256.h"
+#include "util/stringops.h"
+
+#include <string.h>
+#include <SDL.h>
+
+#if SDL_BYTEORDER != SDL_LIL_ENDIAN
+	#error "This SHA-256 is little-endian only"
+#endif
+
+typedef uint32_t WORD;
+typedef uint8_t BYTE;
+
+struct SHA256State {
+	BYTE data[64];
+	WORD datalen;
+	uint64_t bitlen;
+	WORD state[8];
+};
+
+/*
+ * The following code was taken from https://github.com/B-Con/crypto-algorithms/blob/master/sha256.c
+ * with minimal modifications for Taisei.
+ */
+
+/*********************************************************************
+* Filename:   sha256.c
+* Author:     Brad Conte (brad AT bradconte.com)
+* Copyright:
+* Disclaimer: This code is presented "as is" without any guarantees.
+* Details:    Implementation of the SHA-256 hashing algorithm.
+              SHA-256 is one of the three algorithms in the SHA2
+              specification. The others, SHA-384 and SHA-512, are not
+              offered in this implementation.
+              Algorithm specification can be found here:
+               * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf
+              This implementation uses little endian byte order.
+*********************************************************************/
+
+/****************************** MACROS ******************************/
+#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b))))
+#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b))))
+
+#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z)))
+#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
+#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22))
+#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25))
+#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3))
+#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10))
+
+/**************************** VARIABLES *****************************/
+static const WORD k[64] = {
+	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
+	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
+	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
+	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
+	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
+	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
+	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
+	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
+};
+
+/*********************** FUNCTION DEFINITIONS ***********************/
+static void sha256_transform(SHA256State *ctx, const BYTE data[])
+{
+	WORD a, b, c, d, e, f, g, h, i, j, t1, t2, m[64];
+
+	for (i = 0, j = 0; i < 16; ++i, j += 4)
+		m[i] = ((WORD)data[j] << 24) | ((WORD)data[j + 1] << 16) | ((WORD)data[j + 2] << 8) | ((WORD)data[j + 3]);
+	for ( ; i < 64; ++i)
+		m[i] = SIG1(m[i - 2]) + m[i - 7] + SIG0(m[i - 15]) + m[i - 16];
+
+	a = ctx->state[0];
+	b = ctx->state[1];
+	c = ctx->state[2];
+	d = ctx->state[3];
+	e = ctx->state[4];
+	f = ctx->state[5];
+	g = ctx->state[6];
+	h = ctx->state[7];
+
+	for (i = 0; i < 64; ++i) {
+		t1 = h + EP1(e) + CH(e,f,g) + k[i] + m[i];
+		t2 = EP0(a) + MAJ(a,b,c);
+		h = g;
+		g = f;
+		f = e;
+		e = d + t1;
+		d = c;
+		c = b;
+		b = a;
+		a = t1 + t2;
+	}
+
+	ctx->state[0] += a;
+	ctx->state[1] += b;
+	ctx->state[2] += c;
+	ctx->state[3] += d;
+	ctx->state[4] += e;
+	ctx->state[5] += f;
+	ctx->state[6] += g;
+	ctx->state[7] += h;
+}
+
+static void sha256_init(SHA256State *ctx)
+{
+	ctx->datalen = 0;
+	ctx->bitlen = 0;
+	ctx->state[0] = 0x6a09e667;
+	ctx->state[1] = 0xbb67ae85;
+	ctx->state[2] = 0x3c6ef372;
+	ctx->state[3] = 0xa54ff53a;
+	ctx->state[4] = 0x510e527f;
+	ctx->state[5] = 0x9b05688c;
+	ctx->state[6] = 0x1f83d9ab;
+	ctx->state[7] = 0x5be0cd19;
+}
+
+SHA256State* sha256_new(void) {
+	SHA256State *ctx = calloc(1, sizeof(*ctx));
+	sha256_init(ctx);
+	return ctx;
+}
+
+void sha256_free(SHA256State *ctx) {
+	free(ctx);
+}
+
+void sha256_update(SHA256State *ctx, const BYTE data[], size_t len)
+{
+	WORD i;
+
+	for (i = 0; i < len; ++i) {
+		ctx->data[ctx->datalen] = data[i];
+		ctx->datalen++;
+		if (ctx->datalen == 64) {
+			sha256_transform(ctx, ctx->data);
+			ctx->bitlen += 512;
+			ctx->datalen = 0;
+		}
+	}
+}
+
+void sha256_final(SHA256State *ctx, BYTE hash[], size_t hashlen)
+{
+	assert(hashlen >= SHA256_BLOCK_SIZE);
+
+	WORD i;
+
+	i = ctx->datalen;
+
+	// Pad whatever data is left in the buffer.
+	if (ctx->datalen < 56) {
+		ctx->data[i++] = 0x80;
+		while (i < 56)
+			ctx->data[i++] = 0x00;
+	}
+	else {
+		ctx->data[i++] = 0x80;
+		while (i < 64)
+			ctx->data[i++] = 0x00;
+		sha256_transform(ctx, ctx->data);
+		memset(ctx->data, 0, 56);
+	}
+
+	// Append to the padding the total message's length in bits and transform.
+	ctx->bitlen += ctx->datalen * 8;
+	ctx->data[63] = ctx->bitlen;
+	ctx->data[62] = ctx->bitlen >> 8;
+	ctx->data[61] = ctx->bitlen >> 16;
+	ctx->data[60] = ctx->bitlen >> 24;
+	ctx->data[59] = ctx->bitlen >> 32;
+	ctx->data[58] = ctx->bitlen >> 40;
+	ctx->data[57] = ctx->bitlen >> 48;
+	ctx->data[56] = ctx->bitlen >> 56;
+	sha256_transform(ctx, ctx->data);
+
+	// Since this implementation uses little endian byte ordering and SHA uses big endian,
+	// reverse all the bytes when copying the final state to the output hash.
+	for (i = 0; i < 4; ++i) {
+		hash[i]      = (ctx->state[0] >> (24 - i * 8)) & 0x000000ff;
+		hash[i + 4]  = (ctx->state[1] >> (24 - i * 8)) & 0x000000ff;
+		hash[i + 8]  = (ctx->state[2] >> (24 - i * 8)) & 0x000000ff;
+		hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0x000000ff;
+		hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0x000000ff;
+		hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0x000000ff;
+		hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0x000000ff;
+		hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff;
+	}
+}
+
+void sha256_digest(const uint8_t *data, size_t len, uint8_t hash[SHA256_BLOCK_SIZE], size_t hashlen) {
+	assert(hashlen >= SHA256_BLOCK_SIZE);
+
+	SHA256State ctx;
+	sha256_init(&ctx);
+	sha256_update(&ctx, data, len);
+	sha256_final(&ctx, hash, hashlen);
+}
+
+void sha256_hexdigest(const uint8_t *data, size_t len, char hash[SHA256_BLOCK_SIZE*2+1], size_t hashlen) {
+	assert(hashlen >= SHA256_BLOCK_SIZE * 2 + 1);
+
+	uint8_t digest[SHA256_BLOCK_SIZE];
+	sha256_digest(data, len, digest, sizeof(digest));
+	hexdigest(digest, sizeof(digest), hash, hashlen);
+}

src/util/sha256_openssl.c

@@ -0,0 +1,54 @@
+/*
+ * This software is licensed under the terms of the MIT-License
+ * See COPYING for further information.
+ * ---
+ * Copyright (c) 2011-2019, Lukas Weber <laochailan@web.de>.
+ * Copyright (c) 2012-2019, Andrei Alexeyev <akari@alienslab.net>.
+ */
+
+#include "taisei.h"
+
+#include "sha256.h"
+#include "util.h"
+
+#include <openssl/sha.h>
+
+struct SHA256State {
+	SHA256_CTX context;
+};
+
+SHA256State* sha256_new(void) {
+	SHA256State *st = calloc(1, sizeof(*st));
+	SHA256_Init(&st->context);
+	return st;
+}
+
+void sha256_free(SHA256State *ctx) {
+	free(ctx);
+}
+
+void sha256_update(SHA256State *ctx, const uint8_t *data, size_t len) {
+    SHA256_Update(&ctx->context, data, len);
+}
+
+void sha256_final(SHA256State *ctx, uint8_t hash[SHA256_BLOCK_SIZE], size_t hashlen) {
+	assert(hashlen >= SHA256_BLOCK_SIZE);
+	SHA256_Final(hash, &ctx->context);
+}
+
+void sha256_digest(const uint8_t *data, size_t len, uint8_t hash[SHA256_BLOCK_SIZE], size_t hashlen) {
+	assert(hashlen >= SHA256_BLOCK_SIZE);
+
+	SHA256_CTX ctx;
+	SHA256_Init(&ctx);
+	SHA256_Update(&ctx, data, len);
+	SHA256_Final(hash, &ctx);
+}
+
+void sha256_hexdigest(const uint8_t *data, size_t len, char hash[SHA256_BLOCK_SIZE*2+1], size_t hashlen) {
+	assert(hashlen >= SHA256_BLOCK_SIZE * 2 + 1);
+
+	uint8_t digest[SHA256_BLOCK_SIZE];
+	sha256_digest(data, len, digest, sizeof(digest));
+	hexdigest(digest, sizeof(digest), hash, hashlen);
+}

src/util/stringops.c

@@ -455,3 +455,18 @@ void format_huge_num(uint digits, uint num, size_t bufsize, char *buf) {
 	*p = 0;
 	assert(p == buf + len - 1);
 }
+
+void hexdigest(uint8_t *input, size_t input_size, char *output, size_t output_size) {
+	assert(output_size > input_size * 2);
+
+	static char charmap[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
+	uint8_t *end = input + input_size;
+
+	while(input < end) {
+		uint8_t byte = *input++;
+		*output++ = charmap[byte >> 4];
+		*output++ = charmap[byte & 0xf];
+	}
+
+	*output = 0;
+}

src/util/stringops.h

@@ -59,6 +59,7 @@ uint32_t utf8_getch(const char **src) attr_nonnull(1);
 uint32_t crc32str(uint32_t crc, const char *str);
 
 void format_huge_num(uint digits, uint num, size_t bufsize, char *buf);
+void hexdigest(uint8_t *input, size_t input_size, char *output, size_t output_size);
 
 // XXX: Not sure if this is the appropriate header for this
 

src/vfs/setup.c

@@ -26,7 +26,7 @@ static char* get_default_res_path(void) {
 	return res;
 }
 
-static void get_core_paths(char **res, char **storage) {
+static void get_core_paths(char **res, char **storage, char **cache) {
 	if(*(*res = (char*)env_get("TAISEI_RES_PATH", ""))) {
 		*res = strdup(*res);
 	} else {
@@ -38,6 +38,13 @@ static void get_core_paths(char **res, char **storage) {
 	} else {
 		*storage = SDL_GetPrefPath("", "taisei");
 	}
+
+	if(*(*cache = (char*)env_get("TAISEI_CACHE_PATH", ""))) {
+		*cache = strdup(*cache);
+	} else {
+		// TODO: follow XDG on linux
+		*cache = strfmt("%s%ccache", *storage, vfs_get_syspath_separator());
+	}
 }
 
 static bool vfs_mount_pkgdir(const char *dst, const char *src) {
@@ -116,27 +123,33 @@ static void load_packages(const char *dir, const char *unionmp) {
 }
 
 void vfs_setup(bool silent) {
-	char *res_path, *storage_path;
-	get_core_paths(&res_path, &storage_path);
+	char *res_path, *storage_path, *cache_path;
+	get_core_paths(&res_path, &storage_path, &cache_path);
+
+	char *local_res_path = strfmt("%s/resources", storage_path);
 
 	if(!silent) {
 		log_info("Resource path: %s", res_path);
 		log_info("Storage path: %s", storage_path);
+		log_info("Local resource path: %s", local_res_path);
+		log_info("Cache path: %s", cache_path);
 	}
 
-	char *p = NULL;
-
 	struct mpoint_t {
-		const char *dest;   const char *syspath;                      bool loadpaks; uint flags;
+		const char *dest;    const char *syspath; bool loadpaks; uint flags;
 	} mpts[] = {
 		// per-user directory, where configs, replays, screenshots, etc. get stored
-		{"storage",         storage_path,                             false,         VFS_SYSPATH_MOUNT_MKDIR },
+		{ "storage",         storage_path,        false,         VFS_SYSPATH_MOUNT_MKDIR },
 
 		// system-wide directory, contains all of the game assets
-		{"resdirs",         res_path,                                 true,          VFS_SYSPATH_MOUNT_READONLY },
+		{ "resdirs",         res_path,            true,          VFS_SYSPATH_MOUNT_READONLY },
 
 		// subpath of storage, files here override the global assets
-		{"resdirs",         p = strfmt("%s/resources", storage_path), true,          VFS_SYSPATH_MOUNT_MKDIR | VFS_SYSPATH_MOUNT_READONLY },
+		{ "resdirs",         local_res_path,      true,          VFS_SYSPATH_MOUNT_MKDIR | VFS_SYSPATH_MOUNT_READONLY },
+
+		// per-user directory, to contain various cached resources to speed up loading times
+		{ "cache",           cache_path,          false,         VFS_SYSPATH_MOUNT_MKDIR },
+
 		{NULL}
 	};
 
@@ -179,9 +192,10 @@ void vfs_setup(bool silent) {
 	vfs_mkdir_required("storage/replays");
 	vfs_mkdir_required("storage/screenshots");
 
-	free(p);
+	free(local_res_path);
 	free(res_path);
 	free(storage_path);
+	free(cache_path);
 
 	// set up the final "res" union and get rid of the temporaries
 

src/vfs/syspath_public.c

@@ -34,3 +34,7 @@ bool vfs_mount_syspath(const char *mountpoint, const char *fspath, uint flags) {
 
 	return vfs_mount_or_decref(vfs_root, mountpoint, rdir);
 }
+
+char vfs_get_syspath_separator(void) {
+	return vfs_syspath_preferred_separator;
+}

src/vfs/syspath_public.h

@@ -19,4 +19,6 @@ enum {
 bool vfs_mount_syspath(const char *mountpoint, const char *fspath, uint flags)
 	attr_nonnull(1, 2) attr_nodiscard;
 
+char vfs_get_syspath_separator(void);
+
 #endif // IGUARD_vfs_syspath_public_h