diff --git a/.gitmodules b/.gitmodules index 0b167c4f1..7b68e772d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -31,3 +31,6 @@ [submodule "external/fmt"] path = external/fmt url = https://github.com/fmtlib/fmt.git +[submodule "external/SQLiteCpp"] + path = external/SQLiteCpp + url = https://github.com/SRombauts/SQLiteCpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cc47358e0..06dc71d53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -294,6 +294,7 @@ if(NOT MANUAL_SUBMODULES) endif() check_submodule(external/uWebSockets uSockets) check_submodule(external/ghc-filesystem) + check_submodule(external/SQLiteCpp) endif() endif() @@ -504,6 +505,15 @@ if (WITH_SYSTEMD AND NOT BUILD_STATIC_DEPS) endif() +if(BUILD_STATIC_DEPS) + # sqlite3 target already set up +else() + add_library(sqlite3 INTERFACE) + pkg_check_modules(SQLITE3 REQUIRED sqlite3 IMPORTED_TARGET) + message(STATUS "Found sqlite3 ${SQLITE3_VERSION}") + target_link_libraries(sqlite3 INTERFACE PkgConfig::SQLITE3) +endif() + add_subdirectory(external) target_compile_definitions(easylogging PRIVATE AUTO_INITIALIZE_EASYLOGGINGPP) @@ -872,15 +882,6 @@ if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND ARCH_WIDTH EQUAL "32" AND NOT IOS AN endif() -if(BUILD_STATIC_DEPS) - # sqlite3 target already set up -else() - add_library(sqlite3 INTERFACE) - pkg_check_modules(SQLITE3 REQUIRED sqlite3 IMPORTED_TARGET) - message(STATUS "Found sqlite3 ${SQLITE3_VERSION}") - target_link_libraries(sqlite3 INTERFACE PkgConfig::SQLITE3) -endif() - add_subdirectory(contrib) add_subdirectory(src) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 007b8dcc9..4046190a9 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -144,3 +144,9 @@ target_link_libraries(cpr PUBLIC CURL::libcurl) target_include_directories(cpr PUBLIC cpr/include) target_compile_definitions(cpr PUBLIC CPR_CURL_NOSIGNAL) add_library(cpr::cpr ALIAS cpr) + +# disable bundled sqlite3 in sqlitecpp +if(TARGET sqlite3) + option(SQLITECPP_INTERNAL_SQLITE "" OFF) +endif() +add_subdirectory(SQLiteCpp) diff --git a/external/SQLiteCpp b/external/SQLiteCpp new file mode 160000 index 000000000..0c46d86e0 --- /dev/null +++ b/external/SQLiteCpp @@ -0,0 +1 @@ +Subproject commit 0c46d86e0d82031924c5d10bf9b23fa917ba037e diff --git a/src/sqlitedb/database.hpp b/src/sqlitedb/database.hpp new file mode 100644 index 000000000..c90034ab9 --- /dev/null +++ b/src/sqlitedb/database.hpp @@ -0,0 +1,295 @@ +#pragma once + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace db +{ + template + constexpr bool is_cstr = false; + template + constexpr bool is_cstr = true; + template + constexpr bool is_cstr = true; + template <> + constexpr bool is_cstr = true; + template <> + constexpr bool is_cstr = true; + + // Simple wrapper class that can be used to bind a blob through the templated binding code below. + // E.g. `exec_query(st, 100, 42, blob_binder{data})` binds the third parameter using no-copy blob + // binding of the contained data. + struct blob_binder + { + std::string_view data; + explicit blob_binder(std::string_view d) : data{d} + {} + }; + + // Binds a string_view as a no-copy blob at parameter index i. + void + bind_blob_ref(SQLite::Statement& st, int i, std::string_view blob) + { + st.bindNoCopy(i, static_cast(blob.data()), blob.size()); + } + + namespace detail + { + template + void + bind_oneshot_single(SQLite::Statement& st, int i, const T& val) + { + if constexpr (std::is_same_v || is_cstr) + st.bindNoCopy(i, val); + else if constexpr (std::is_same_v) + bind_blob_ref(st, i, val.data); + else + st.bind(i, val); + } + + template + void + bind_oneshot(SQLite::Statement& st, std::integer_sequence, const T&... bind) + { + (bind_oneshot_single(st, Index + 1, bind), ...); + } + } // namespace detail + + // Called from exec_query and similar to bind statement parameters for immediate execution. + // strings (and c strings) use no-copy binding; integer values are bound by value. You can bind a + // blob (by reference, like strings) by passing `blob_binder{data}`. + template + void + bind_oneshot(SQLite::Statement& st, const T&... bind) + { + detail::bind_oneshot(st, std::make_integer_sequence{}, bind...); + } + + // Executes a query that does not expect results. Optionally binds parameters, if provided. + // Returns the number of affected rows; throws on error or if results are returned. + template + int + exec_query(SQLite::Statement& st, const T&... bind) + { + bind_oneshot(st, bind...); + return st.exec(); + } + + // Same as above, but prepares a literal query on the fly for use with queries that are only used + // once. + template + int + exec_query(SQLite::Database& db, const char* query, const T&... bind) + { + SQLite::Statement st{db, query}; + return exec_query(st, bind...); + } + + template + struct first_type + { + using type = T; + }; + template + using first_type_t = typename first_type::type; + + template + using type_or_tuple = std::conditional_t, std::tuple>; + + // Retrieves a single row of values from the current state of a statement (i.e. after a + // executeStep() call that is expecting a return value). If `T...` is a single type then this + // returns the single T value; if T... has multiple types then you get back a tuple of values. + template + T + get(SQLite::Statement& st) + { + return static_cast(st.getColumn(0)); + } + template + std::tuple + get(SQLite::Statement& st) + { + return st.getColumns, 2 + sizeof...(Tn)>(); + } + + // Steps a statement to completion that is expected to return at most one row, optionally binding + // values into it (if provided). Returns a filled out optional (or optional>) + // if a row was retrieved, otherwise a nullopt. Throws if more than one row is retrieved. + template + std::optional> + exec_and_maybe_get(SQLite::Statement& st, const Args&... bind) + { + int i = 1; + (bind_oneshot(st, i, bind), ...); + std::optional> result; + while (st.executeStep()) + { + if (result) + { + MERROR("Expected single-row result, got multiple rows from {}" << st.getQuery()); + throw std::runtime_error{"DB error: expected single-row result, got multiple rows"}; + } + result = get(st); + } + return result; + } + + // Executes a statement to completion that is expected to return exactly one row, optionally + // binding values into it (if provided). Returns a T or std::tuple (depending on whether or + // not more than one T is provided) for the row. Throws an exception if no rows or more than one + // row are returned. + template + type_or_tuple + exec_and_get(SQLite::Statement& st, const Args&... bind) + { + auto maybe_result = exec_and_maybe_get(st, bind...); + if (!maybe_result) + { + MERROR("Expected single-row result, got no rows from {}" << st.getQuery()); + throw std::runtime_error{"DB error: expected single-row result, got not rows"}; + } + return *std::move(maybe_result); + } + + // Executes a query to completion, collecting each row into a vector (or vector> if + // multiple T are given). Can optionally bind before executing. + template + std::vector> + get_all(SQLite::Statement& st, const Bind&... bind) + { + int i = 1; + (bind_oneshot(st, i, bind), ...); + std::vector> results; + while (st.executeStep()) + results.push_back(get(st)); + return results; + } + + // Takes a query prefix and suffix and places ? separated by commas between them + // Example: multi_in_query("foo(", 3, ")bar") will return "foo(?,?,?)bar" + inline std::string + multi_in_query(std::string_view prefix, size_t count, std::string_view suffix) + { + std::string query; + query.reserve(prefix.size() + (count == 0 ? 0 : 2 * count - 1) + suffix.size()); + query += prefix; + for (size_t i = 0; i < count; i++) + { + if (i > 0) + query += ','; + query += '?'; + } + query += suffix; + return query; + } + + // Storage database class. + class Database + { + // SQLiteCpp's statements are not thread-safe, so we prepare them thread-locally when needed + std::unordered_map> + prepared_sts; + std::shared_mutex prepared_sts_mutex; + + /** Wrapper around a SQLite::Statement that calls `tryReset()` on destruction of the wrapper. */ + class StatementWrapper + { + SQLite::Statement& st; + + public: + /// Whether we should reset on destruction; can be set to false if needed. + bool reset_on_destruction = true; + + explicit StatementWrapper(SQLite::Statement& st) noexcept : st{st} + {} + ~StatementWrapper() noexcept + { + if (reset_on_destruction) + st.tryReset(); + } + SQLite::Statement& + operator*() noexcept + { + return st; + } + SQLite::Statement* + operator->() noexcept + { + return &st; + } + operator SQLite::Statement&() noexcept + { + return st; + } + }; + + public: + SQLite::Database db; + + StatementWrapper + prepared_st(const std::string& query) + { + std::unordered_map* sts; + { + std::shared_lock rlock{prepared_sts_mutex}; + if (auto it = prepared_sts.find(std::this_thread::get_id()); it != prepared_sts.end()) + sts = &it->second; + else + { + rlock.unlock(); + std::unique_lock wlock{prepared_sts_mutex}; + sts = &prepared_sts.try_emplace(std::this_thread::get_id()).first->second; + } + } + if (auto qit = sts->find(query); qit != sts->end()) + return StatementWrapper{qit->second}; + return StatementWrapper{sts->try_emplace(query, db, query).first->second}; + } + + template + int + prepared_exec(const std::string& query, const T&... bind) + { + return exec_query(prepared_st(query), bind...); + } + + template + auto + prepared_get(const std::string& query, const Bind&... bind) + { + return exec_and_get(prepared_st(query), bind...); + } + + explicit Database(const std::filesystem::path& db_path, const std::string_view db_password) + : db{db_path, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE | SQLite::OPEN_FULLMUTEX} + { + // Don't fail on these because we can still work even if they fail + if (int rc = db.tryExec("PRAGMA journal_mode = WAL"); rc != SQLITE_OK) + MERROR("Failed to set journal mode to WAL: {}" << sqlite3_errstr(rc)); + + if (int rc = db.tryExec("PRAGMA synchronous = NORMAL"); rc != SQLITE_OK) + MERROR("Failed to set synchronous mode to NORMAL: {}" << sqlite3_errstr(rc)); + + // FIXME: SQLite / SQLiteCPP may not have encryption available + // so this may fail, or worse silently fail and do nothing + if (not db_password.empty()) + { + db.key(std::string{db_password}); + } + } + + ~Database() = default; + }; + +} // namespace db