oxen-io
/
oxen-mq
mirror of https://github.com/oxen-io/oxen-mq.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'dev' into stable

This commit is contained in:
Jason Rhinelander 2021-01-14 15:32:59 -04:00
parent e7487fd0c8 396f591fae
commit 46c1a97b14
53 changed files with 4484 additions and 4276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
CMakeLists.txt
View File

@ -1,65 +1,83 @@
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
find_program(CCACHE_PROGRAM ccache)
if(CCACHE_PROGRAM)
foreach(lang C CXX)
if(NOT DEFINED CMAKE_${lang}_COMPILER_LAUNCHER AND NOT CMAKE_${lang}_COMPILER MATCHES ".*/ccache")
message(STATUS "Enabling ccache for ${lang}")
set(CMAKE_${lang}_COMPILER_LAUNCHER ${CCACHE_PROGRAM} CACHE STRING "")
endif()
endforeach()
endif()
cmake_minimum_required(VERSION 3.7)
# Has to be set before `project()`, and ignored on non-macos:
set(CMAKE_OSX_DEPLOYMENT_TARGET 10.12 CACHE STRING "macOS deployment target (Apple clang only)")
project(liblokimq CXX C)
project(liboxenmq CXX C)
include(GNUInstallDirs)
set(LOKIMQ_VERSION_MAJOR 1)
set(LOKIMQ_VERSION_MINOR 2)
set(LOKIMQ_VERSION_PATCH 2)
set(LOKIMQ_VERSION "${LOKIMQ_VERSION_MAJOR}.${LOKIMQ_VERSION_MINOR}.${LOKIMQ_VERSION_PATCH}")
message(STATUS "lokimq v${LOKIMQ_VERSION}")
set(OXENMQ_VERSION_MAJOR 1)
set(OXENMQ_VERSION_MINOR 2)
set(OXENMQ_VERSION_PATCH 3)
set(OXENMQ_VERSION "${OXENMQ_VERSION_MAJOR}.${OXENMQ_VERSION_MINOR}.${OXENMQ_VERSION_PATCH}")
message(STATUS "oxenmq v${OXENMQ_VERSION}")
set(LOKIMQ_LIBVERSION 0)
set(OXENMQ_LIBVERSION 0)
if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
set(lokimq_IS_TOPLEVEL_PROJECT TRUE)
set(oxenmq_IS_TOPLEVEL_PROJECT TRUE)
else()
set(lokimq_IS_TOPLEVEL_PROJECT FALSE)
set(oxenmq_IS_TOPLEVEL_PROJECT FALSE)
endif()
option(BUILD_SHARED_LIBS "Build shared libraries instead of static ones" ON)
set(lokimq_INSTALL_DEFAULT OFF)
if(BUILD_SHARED_LIBS OR lokimq_IS_TOPLEVEL_PROJECT)
set(lokimq_INSTALL_DEFAULT ON)
set(oxenmq_INSTALL_DEFAULT OFF)
if(BUILD_SHARED_LIBS OR oxenmq_IS_TOPLEVEL_PROJECT)
set(oxenmq_INSTALL_DEFAULT ON)
endif()
option(LOKIMQ_BUILD_TESTS "Building and perform lokimq tests" ${lokimq_IS_TOPLEVEL_PROJECT})
option(LOKIMQ_INSTALL "Add lokimq libraries and headers to cmake install target; defaults to ON if BUILD_SHARED_LIBS is enabled or we are the top-level project; OFF for a static subdirectory build" ${lokimq_INSTALL_DEFAULT})
option(LOKIMQ_INSTALL_CPPZMQ "Install cppzmq header with lokimq/ headers (requires LOKIMQ_INSTALL)" ON)
option(OXENMQ_BUILD_TESTS "Building and perform oxenmq tests" ${oxenmq_IS_TOPLEVEL_PROJECT})
option(OXENMQ_INSTALL "Add oxenmq libraries and headers to cmake install target; defaults to ON if BUILD_SHARED_LIBS is enabled or we are the top-level project; OFF for a static subdirectory build" ${oxenmq_INSTALL_DEFAULT})
option(OXENMQ_INSTALL_CPPZMQ "Install cppzmq header with oxenmq/ headers (requires OXENMQ_INSTALL)" ON)
option(OXENMQ_LOKIMQ_COMPAT "Install lokimq compatibility headers and pkg-config for rename migration" ON)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
configure_file(lokimq/version.h.in lokimq/version.h @ONLY)
configure_file(liblokimq.pc.in liblokimq.pc @ONLY)
configure_file(oxenmq/version.h.in oxenmq/version.h @ONLY)
configure_file(liboxenmq.pc.in liboxenmq.pc @ONLY)
if(OXENMQ_LOKIMQ_COMPAT)
configure_file(liblokimq.pc.in liblokimq.pc @ONLY)
endif()
add_library(lokimq
lokimq/address.cpp
lokimq/auth.cpp
lokimq/bt_serialize.cpp
lokimq/connections.cpp
lokimq/jobs.cpp
lokimq/lokimq.cpp
lokimq/proxy.cpp
lokimq/worker.cpp
add_library(oxenmq
oxenmq/address.cpp
oxenmq/auth.cpp
oxenmq/bt_serialize.cpp
oxenmq/connections.cpp
oxenmq/jobs.cpp
oxenmq/oxenmq.cpp
oxenmq/proxy.cpp
oxenmq/worker.cpp
)
set_target_properties(lokimq PROPERTIES SOVERSION ${LOKIMQ_LIBVERSION})
set_target_properties(oxenmq PROPERTIES SOVERSION ${OXENMQ_LIBVERSION})
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
target_link_libraries(lokimq PRIVATE Threads::Threads)
target_link_libraries(oxenmq PRIVATE Threads::Threads)
# libzmq is nearly impossible to link statically from a system-installed static library: it depends
# on a ton of other libraries, some of which are not all statically available. If the caller wants
# to mess with this, so be it: they can set up a libzmq target and we'll use it. Otherwise if they
# asked us to do things statically, don't even try to find a system lib and just build it.
set(lokimq_build_static_libzmq OFF)
set(oxenmq_build_static_libzmq OFF)
if(TARGET libzmq)
target_link_libraries(lokimq PUBLIC libzmq)
target_link_libraries(oxenmq PUBLIC libzmq)
elseif(BUILD_SHARED_LIBS)
include(FindPkgConfig)
pkg_check_modules(libzmq libzmq>=4.3 IMPORTED_TARGET)
@ -75,30 +93,30 @@ elseif(BUILD_SHARED_LIBS)
set_property(TARGET PkgConfig::libzmq PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${zmq_inc})
endif()
target_link_libraries(lokimq PUBLIC PkgConfig::libzmq)
target_link_libraries(oxenmq PUBLIC PkgConfig::libzmq)
else()
set(lokimq_build_static_libzmq ON)
set(oxenmq_build_static_libzmq ON)
endif()
else()
set(lokimq_build_static_libzmq ON)
set(oxenmq_build_static_libzmq ON)
endif()
if(lokimq_build_static_libzmq)
if(oxenmq_build_static_libzmq)
message(STATUS "libzmq >= 4.3 not found or static build requested, building bundled version")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/local-libzmq")
include(LocalLibzmq)
target_link_libraries(lokimq PUBLIC libzmq_vendor)
target_link_libraries(oxenmq PUBLIC libzmq_vendor)
endif()
target_include_directories(lokimq
target_include_directories(oxenmq
PUBLIC
$<INSTALL_INTERFACE:>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cppzmq>
)
target_compile_options(lokimq PRIVATE -Wall -Wextra -Werror)
set_target_properties(lokimq PROPERTIES
target_compile_options(oxenmq PRIVATE -Wall -Wextra -Werror)
set_target_properties(oxenmq PROPERTIES
CXX_STANDARD 17
CXX_STANDARD_REQUIRED ON
CXX_EXTENSIONS OFF
@ -117,8 +135,8 @@ endfunction()
# If the caller has already set up a sodium target then we will just link to it, otherwise we go
# looking for it.
if(TARGET sodium)
target_link_libraries(lokimq PUBLIC sodium)
if(lokimq_build_static_libzmq)
target_link_libraries(oxenmq PUBLIC sodium)
if(oxenmq_build_static_libzmq)
target_link_libraries(libzmq_vendor INTERFACE sodium)
endif()
else()
@ -126,67 +144,95 @@ else()
pkg_check_modules(sodium REQUIRED libsodium IMPORTED_TARGET)
if(BUILD_SHARED_LIBS)
target_link_libraries(lokimq PUBLIC PkgConfig::sodium)
if(lokimq_build_static_libzmq)
target_link_libraries(oxenmq PUBLIC PkgConfig::sodium)
if(oxenmq_build_static_libzmq)
target_link_libraries(libzmq_vendor INTERFACE PkgConfig::sodium)
endif()
else()
link_dep_libs(lokimq PUBLIC "${sodium_STATIC_LIBRARY_DIRS}" ${sodium_STATIC_LIBRARIES})
target_include_directories(lokimq PUBLIC ${sodium_STATIC_INCLUDE_DIRS})
if(lokimq_build_static_libzmq)
link_dep_libs(oxenmq PUBLIC "${sodium_STATIC_LIBRARY_DIRS}" ${sodium_STATIC_LIBRARIES})
target_include_directories(oxenmq PUBLIC ${sodium_STATIC_INCLUDE_DIRS})
if(oxenmq_build_static_libzmq)
link_dep_libs(libzmq_vendor INTERFACE "${sodium_STATIC_LIBRARY_DIRS}" ${sodium_STATIC_LIBRARIES})
target_link_libraries(libzmq_vendor INTERFACE ${sodium_STATIC_INCLUDE_DIRS})
endif()
endif()
endif()
add_library(lokimq::lokimq ALIAS lokimq)
add_library(oxenmq::oxenmq ALIAS oxenmq)
if(OXENMQ_LOKIMQ_COMPAT)
add_library(lokimq::lokimq ALIAS oxenmq)
endif()
export(
TARGETS lokimq
NAMESPACE lokimq::
FILE lokimqTargets.cmake
TARGETS oxenmq
NAMESPACE oxenmq::
FILE oxenmqTargets.cmake
)
if(LOKIMQ_INSTALL)
if(OXENMQ_INSTALL)
install(
TARGETS lokimq
EXPORT lokimqConfig
TARGETS oxenmq
EXPORT oxenmqConfig
DESTINATION ${CMAKE_INSTALL_LIBDIR}
)
install(
FILES lokimq/address.h
lokimq/auth.h
lokimq/base32z.h
lokimq/base64.h
lokimq/batch.h
lokimq/bt_serialize.h
lokimq/bt_value.h
lokimq/connections.h
lokimq/hex.h
lokimq/lokimq.h
lokimq/message.h
lokimq/string_view.h
lokimq/variant.h
${CMAKE_CURRENT_BINARY_DIR}/lokimq/version.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lokimq
FILES oxenmq/address.h
oxenmq/auth.h
oxenmq/base32z.h
oxenmq/base64.h
oxenmq/batch.h
oxenmq/bt_serialize.h
oxenmq/bt_value.h
oxenmq/connections.h
oxenmq/hex.h
oxenmq/oxenmq.h
oxenmq/message.h
oxenmq/variant.h
${CMAKE_CURRENT_BINARY_DIR}/oxenmq/version.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/oxenmq
)
if(LOKIMQ_INSTALL_CPPZMQ)
if(OXENMQ_INSTALL_CPPZMQ)
install(
FILES cppzmq/zmq.hpp
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lokimq
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/oxenmq
)
endif()
install(
FILES ${CMAKE_CURRENT_BINARY_DIR}/liblokimq.pc
FILES ${CMAKE_CURRENT_BINARY_DIR}/liboxenmq.pc
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig
)
if(OXENMQ_LOKIMQ_COMPAT)
install(
FILES lokimq/address.h
lokimq/auth.h
lokimq/base32z.h
lokimq/base64.h
lokimq/batch.h
lokimq/bt_serialize.h
lokimq/bt_value.h
lokimq/connections.h
lokimq/hex.h
lokimq/lokimq.h
lokimq/message.h
lokimq/variant.h
lokimq/version.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lokimq
)
install(
FILES ${CMAKE_CURRENT_BINARY_DIR}/liblokimq.pc
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig
)
endif()
endif()
if(LOKIMQ_BUILD_TESTS)
if(OXENMQ_BUILD_TESTS)
add_subdirectory(tests)
endif()

6
liblokimq.pc.in
View File

@ -4,10 +4,10 @@ libdir=@CMAKE_INSTALL_FULL_LIBDIR@
includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
Name: liblokimq
Description: ZeroMQ-based communication library for Loki
Version: @LOKIMQ_VERSION@
Description: ZeroMQ-based communication library (compatibility package for liboxenmq)
Version: @OXENMQ_VERSION@
Libs: -L${libdir} -llokimq
Libs: -L${libdir} -loxenmq
Libs.private: @PRIVATE_LIBS@
Requires.private: libzmq libsodium
Cflags: -I${includedir}

13
liboxenmq.pc.in Normal file
View File

@ -0,0 +1,13 @@
prefix=@CMAKE_INSTALL_PREFIX@
exec_prefix=${prefix}
libdir=@CMAKE_INSTALL_FULL_LIBDIR@
includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
Name: liboxenmq
Description: ZeroMQ-based communication library
Version: @OXENMQ_VERSION@
Libs: -L${libdir} -loxenmq
Libs.private: @PRIVATE_LIBS@
Requires.private: libzmq libsodium
Cflags: -I${includedir}

210
lokimq/address.h
View File

@ -1,210 +1,4 @@
// Copyright (c) 2020, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <string>
#include <string_view>
#include <cstdint>
#include <iosfwd>
#include "../oxenmq/address.h"
namespace lokimq {
using namespace std::literals;
/** LokiMQ address abstraction class. This class uses and extends standard ZMQ addresses allowing
* extra parameters to be passed in in a relative standard way.
*
* External ZMQ addresses generally have two forms that we are concerned with: one for TCP and one
* for Unix sockets:
*
* tcp://HOST:PORT -- HOST can be a hostname, IPv4 address, or IPv6 address in [...]
* ipc://PATH -- PATH can be absolute (ipc:///path/to/some.sock) or relative (ipc://some.sock)
*
* but this doesn't carry enough info: in particular, we can connect with two very different
* protocols: curve25519-encrypted, or plaintext, but for curve25519-encrypted we require the
* remote's public key as well to verify the connection.
*
* This class, then, handles this by allowing addresses of:
*
* Standard ZMQ address: these carry no pubkey and so the connection will be unencrypted:
*
* tcp://HOSTNAME:PORT
* ipc://PATH
*
* Non-ZMQ address formats that specify that the connection shall be x25519 encrypted:
*
* curve://HOSTNAME:PORT/PUBKEY -- PUBKEY must be specified in hex (64 characters), base32z (52)
* or base64 (43 or 44 with one '=' trailing padding)
* ipc+curve:///path/to/my.sock/PUBKEY -- same requirements on PUBKEY as above.
* tcp+curve://(whatever) -- alias for curve://(whatever)
*
* We also accept special upper-case TCP-only variants which *only* accept uppercase characters and
* a few required symbols (:, /, $, ., and -) in the string:
*
* TCP://HOSTNAME:PORT
* CURVE://HOSTNAME:PORT/B32ZPUBKEY
*
* These versions are explicitly meant to be used with QR codes; the upper-case-only requirement
* allows a smaller QR code by allowing QR's alphanumeric mode (which allows only [A-Z0-9 $%*+./:-])
* to be used. Such a QR-friendly address can be created from the qr_address() method. To support
* literal IPv6 addresses we surround the address with $...$ instead of the usual [...].
*
* Note that this class does very little validate the host argument at all, and no socket path
* validation whatsoever. The only constraint on host is when parsing an encoded address: we check
* that it contains no : at all, or must be a [bracketed] expression that contains only hex
* characters, :'s, or .'s. Otherwise, if you pass broken crap into the hostname, expect broken
* crap out.
*/
struct address {
/// Supported address protocols: TCP connections (tcp), or unix sockets (ipc).
enum class proto {
tcp,
tcp_curve,
ipc,
ipc_curve
};
/// Supported public key encodings (used when regenerating an augmented address).
enum class encoding {
hex, ///< hexadecimal encoded
base32z, ///< base32z encoded
base64, ///< base64 encoded (*without* trailing = padding)
BASE32Z ///< upper-case base32z encoding, meant for QR encoding
};
/// The protocol: one of the `protocol` enum values for tcp or ipc (unix sockets), with or
/// without _curve encryption.
proto protocol = proto::tcp;
/// The host for tcp connections; can be a hostname or IP address. If this is an IPv6 it must be surrounded with [ ].
std::string host;
/// The port (for tcp connections)
uint16_t port = 0;
/// The socket path (for unix socket connections)
std::string socket;
/// If a curve connection, this is the required remote public key (in bytes)
std::string pubkey;
/// Default constructor; this gives you an unusable address.
address() = default;
/**
* Constructs an address by parsing a string_view containing one of the formats listed in the
* class description. This is intentionally implicitly constructible so that you can pass a
* string_view into anything expecting an `address`.
*
* Throw std::invalid_argument if the given address is not parseable.
*/
address(std::string_view addr);
/** Constructs an address from a remote string and a separate pubkey. Typically `remote` is a
* basic ZMQ connect string, though this is not enforced. Any pubkey information embedded in
* the remote string will be discarded and replaced with the given pubkey string. The result
* will be curve encrypted if `pubkey` is non-empty, plaintext if `pubkey` is empty.
*
* Throws an exception if either addr or pubkey is invalid.
*
* Exactly equivalent to `address a{remote}; a.set_pubkey(pubkey);`
*/
address(std::string_view addr, std::string_view pubkey) : address(addr) { set_pubkey(pubkey); }
/// Replaces the address's pubkey (if any) with the given pubkey (or no pubkey if empty). If
/// changing from pubkey to no-pubkey or no-pubkey to pubkey then the protocol is update to
/// switch to or from curve encryption.
///
/// pubkey should be the 32-byte binary pubkey, or an empty string to remove an existing pubkey.
///
/// Returns the object itself, so that you can chain it.
address& set_pubkey(std::string_view pubkey);
/// Constructs and builds the ZMQ connection address from the stored connection details. This
/// does not contain any of the curve-related details; those must be specified separately when
/// interfacing with ZMQ.
std::string zmq_address() const;
/// Returns true if the connection was specified as a curve-encryption-enabled connection, false
/// otherwise.
bool curve() const { return protocol == proto::tcp_curve || protocol == proto::ipc_curve; }
/// True if the protocol is TCP (either with or without curve)
bool tcp() const { return protocol == proto::tcp || protocol == proto::tcp_curve; }
/// True if the protocol is unix socket (either with or without curve)
bool ipc() const { return !tcp(); }
/// Returns the full "augmented" address string (i.e. that could be passed in to the
/// constructor). This will be equivalent (but not necessarily identical) to an augmented
/// string passed into the constructor. Takes an optional encoding format for the pubkey (if
/// any), which defaults to base32z.
std::string full_address(encoding enc = encoding::base32z) const;
/// Returns a QR-code friendly address string. This returns an all-uppercase version of the
/// address with "TCP://" or "CURVE://" for the protocol string, and uses upper-case base32z
/// encoding for the pubkey (for curve addresses). For literal IPv6 addresses we replace the
/// surround the
/// address with $ instead of $
///
/// \throws std::logic_error if called on a unix socket address.
std::string qr_address() const;
/// Returns `.pubkey` but encoded in the given format
std::string encode_pubkey(encoding enc) const;
/// Returns true if two addresses are identical (i.e. same protocol and relevant protocol
/// arguments).
///
/// Note that it is possible for addresses to connect to the same socket without being
/// identical: for example, using "foo.sock" and "./foo.sock", or writing IPv6 addresses (or
/// even IPv4 addresses) in slightly different ways). Such equivalent but non-equal values will
/// result in a false return here.
///
/// Note also that we ignore irrelevant arguments: for example, we don't care whether pubkeys
/// match when comparing two non-curve TCP addresses.
bool operator==(const address& other) const;
/// Negation of ==
bool operator!=(const address& other) const { return !operator==(other); }
/// Factory function that constructs a TCP address from a host and port. The connection will be
/// plaintext. If the host is an IPv6 address it *must* be surrounded with [ and ].
static address tcp(std::string host, uint16_t port);
/// Factory function that constructs a curve-encrypted TCP address from a host, port, and remote
/// pubkey. The pubkey must be 32 bytes. As above, IPv6 addresses must be specified as [addr].
static address tcp_curve(std::string host, uint16_t, std::string pubkey);
/// Factory function that constructs a unix socket address from a path. The connection will be
/// plaintext (which is usually fine for a socket since unix sockets are local machine).
static address ipc(std::string path);
/// Factory function that constructs a unix socket address from a path and remote pubkey. The
/// connection will be curve25519 encrypted; the remote pubkey must be 32 bytes.
static address ipc_curve(std::string path, std::string pubkey);
};
// Outputs address.full_address() when sent to an ostream.
std::ostream& operator<<(std::ostream& o, const address& a);
}
namespace lokimq = oxenmq;

55
lokimq/auth.h
View File

@ -1,55 +1,4 @@
#pragma once
#include <iosfwd>
#include <string>
#include <cstring>
#include <unordered_set>
#include "../oxenmq/auth.h"
namespace lokimq {
/// Authentication levels for command categories and connections
enum class AuthLevel {
denied, ///< Not actually an auth level, but can be returned by the AllowFunc to deny an incoming connection.
none, ///< No authentication at all; any random incoming ZMQ connection can invoke this command.
basic, ///< Basic authentication commands require a login, or a node that is specifically configured to be a public node (e.g. for public RPC).
admin, ///< Advanced authentication commands require an admin user, either via explicit login or by implicit login from localhost. This typically protects administrative commands like shutting down, starting mining, or access sensitive data.
};
std::ostream& operator<<(std::ostream& os, AuthLevel a);
/// The access level for a command category
struct Access {
/// Minimum access level required
AuthLevel auth;
/// If true only remote SNs may call the category commands
bool remote_sn;
/// If true the category requires that the local node is a SN
bool local_sn;
/// Constructor. Intentionally allows implicit conversion from an AuthLevel so that an
/// AuthLevel can be passed anywhere an Access is required (the resulting Access will have both
/// remote and local sn set to false).
Access(AuthLevel auth = AuthLevel::none, bool remote_sn = false, bool local_sn = false)
: auth{auth}, remote_sn{remote_sn}, local_sn{local_sn} {}
};
/// Simple hash implementation for a string that is *already* a hash-like value (such as a pubkey).
/// Falls back to std::hash<std::string> if given a string smaller than a size_t.
struct already_hashed {
size_t operator()(const std::string& s) const {
if (s.size() < sizeof(size_t))
return std::hash<std::string>{}(s);
size_t hash;
std::memcpy(&hash, &s[0], sizeof(hash));
return hash;
}
};
/// std::unordered_set specialization for specifying pubkeys (used, in particular, by
/// LokiMQ::set_active_sns and LokiMQ::update_active_sns); this is a std::string unordered_set that
/// also uses a specialized trivial hash function that uses part of the value itself (i.e. the
/// pubkey) directly as a hash value. (This is nice and fast for uniformly distributed values like
/// pubkeys and a terrible hash choice for anything else).
using pubkey_set = std::unordered_set<std::string, already_hashed>;
}
namespace lokimq = oxenmq;

203
lokimq/base32z.h
View File

@ -1,203 +1,4 @@
// Copyright (c) 2019-2020, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <string>
#include <string_view>
#include <array>
#include <iterator>
#include <cassert>
#include "../oxenmq/base32z.h"
namespace lokimq {
namespace detail {
/// Compile-time generated lookup tables for base32z conversion. This is case insensitive (though
/// for byte -> b32z conversion we always produce lower case).
struct b32z_table {
// Store the 0-31 decoded value of every possible char; all the chars that aren't valid are set
// to 0. (If you don't trust your data, check it with is_base32z first, which uses these 0's
// to detect invalid characters -- which is why we want a full 256 element array).
char from_b32z_lut[256];
// Store the encoded character of every 0-31 (5 bit) value.
char to_b32z_lut[32];
// constexpr constructor that fills out the above (and should do it at compile time for any half
// decent compiler).
constexpr b32z_table() noexcept : from_b32z_lut{},
to_b32z_lut{
'y', 'b', 'n', 'd', 'r', 'f', 'g', '8', 'e', 'j', 'k', 'm', 'c', 'p', 'q', 'x',
'o', 't', '1', 'u', 'w', 'i', 's', 'z', 'a', '3', '4', '5', 'h', '7', '6', '9'
}
{
for (unsigned char c = 0; c < 32; c++) {
unsigned char x = to_b32z_lut[c];
from_b32z_lut[x] = c;
if (x >= 'a' && x <= 'z')
from_b32z_lut[x - 'a' + 'A'] = c;
}
}
// Convert a b32z encoded character into a 0-31 value
constexpr char from_b32z(unsigned char c) const noexcept { return from_b32z_lut[c]; }
// Convert a 0-31 value into a b32z encoded character
constexpr char to_b32z(unsigned char b) const noexcept { return to_b32z_lut[b]; }
} constexpr b32z_lut;
// This main point of this static assert is to force the compiler to compile-time build the constexpr tables.
static_assert(b32z_lut.from_b32z('w') == 20 && b32z_lut.from_b32z('T') == 17 && b32z_lut.to_b32z(5) == 'f', "");
} // namespace detail
/// Converts bytes into a base32z encoded character sequence.
template <typename InputIt, typename OutputIt>
void to_base32z(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "to_base32z requires chars/bytes");
int bits = 0; // Tracks the number of unconsumed bits held in r, will always be in [0, 4]
std::uint_fast16_t r = 0;
while (begin != end) {
r = r << 8 | static_cast<unsigned char>(*begin++);
// we just added 8 bits, so we can *always* consume 5 to produce one character, so (net) we
// are adding 3 bits.
bits += 3;
*out++ = detail::b32z_lut.to_b32z(r >> bits); // Right-shift off the bits we aren't consuming right now
// Drop the bits we don't want to keep (because we just consumed them)
r &= (1 << bits) - 1;
if (bits >= 5) { // We have enough bits to produce a second character; essentially the same as above
bits -= 5; // Except now we are just consuming 5 without having added any more
*out++ = detail::b32z_lut.to_b32z(r >> bits);
r &= (1 << bits) - 1;
}
}
if (bits > 0) // We hit the end, but still have some unconsumed bits so need one final character to append
*out++ = detail::b32z_lut.to_b32z(r << (5 - bits));
}
/// Creates a base32z string from an iterator pair of a byte sequence.
template <typename It>
std::string to_base32z(It begin, It end) {
std::string base32z;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>)
base32z.reserve((std::distance(begin, end)*8 + 4) / 5); // == bytes*8/5, rounded up.
to_base32z(begin, end, std::back_inserter(base32z));
return base32z;
}
/// Creates a base32z string from an iterable, std::string-like object
template <typename CharT>
std::string to_base32z(std::basic_string_view<CharT> s) { return to_base32z(s.begin(), s.end()); }
inline std::string to_base32z(std::string_view s) { return to_base32z<>(s); }
/// Returns true if all elements in the range are base32z characters
template <typename It>
constexpr bool is_base32z(It begin, It end) {
static_assert(sizeof(decltype(*begin)) == 1, "is_base32z requires chars/bytes");
for (; begin != end; ++begin) {
auto c = static_cast<unsigned char>(*begin);
if (detail::b32z_lut.from_b32z(c) == 0 && !(c == 'y' || c == 'Y'))
return false;
}
return true;
}
/// Returns true if all elements in the string-like value are base32z characters
template <typename CharT>
constexpr bool is_base32z(std::basic_string_view<CharT> s) { return is_base32z(s.begin(), s.end()); }
constexpr bool is_base32z(std::string_view s) { return is_base32z<>(s); }
/// Converts a sequence of base32z digits to bytes. Undefined behaviour if any characters are not
/// valid base32z alphabet characters. It is permitted for the input and output ranges to overlap
/// as long as `out` is no earlier than `begin`. Note that if you pass in a sequence that could not
/// have been created by a base32z encoding of a byte sequence, we treat the excess bits as if they
/// were not provided.
///
/// For example, "yyy" represents a 15-bit value, but a byte sequence is either 8-bit (requiring 2
/// characters) or 16-bit (requiring 4). Similarly, "yb" is an impossible encoding because it has
/// its 10th bit set (b = 00001), but a base32z encoded value should have all 0's beyond the 8th (or
/// 16th or 24th or ... bit). We treat any such bits as if they were not specified (even if they
/// are): which means "yy", "yb", "yyy", "yy9", "yd", etc. all decode to the same 1-byte value "\0".
template <typename InputIt, typename OutputIt>
void from_base32z(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "from_base32z requires chars/bytes");
uint_fast16_t curr = 0;
int bits = 0; // number of bits we've loaded into val; we always keep this < 8.
while (begin != end) {
curr = curr << 5 | detail::b32z_lut.from_b32z(static_cast<unsigned char>(*begin++));
if (bits >= 3) {
bits -= 3; // Added 5, removing 8
*out++ = static_cast<uint8_t>(curr >> bits);
curr &= (1 << bits) - 1;
} else {
bits += 5;
}
}
// Ignore any trailing bits. base32z encoding always has at least as many bits as the source
// bytes, which means we should not be able to get here from a properly encoded b32z value with
// anything other than 0s: if we have no extra bits (e.g. 5 bytes == 8 b32z chars) then we have
// a 0-bit value; if we have some extra bits (e.g. 6 bytes requires 10 b32z chars, but that
// contains 50 bits > 48 bits) then those extra bits will be 0s (and this covers the bits -= 3
// case above: it'll leave us with 0-4 extra bits, but those extra bits would be 0 if produced
// from an actual byte sequence).
//
// The "bits += 5" case, then, means that we could end with 5-7 bits. This, however, cannot be
// produced by a valid encoding:
// - 0 bytes gives us 0 chars with 0 leftover bits
// - 1 byte gives us 2 chars with 2 leftover bits
// - 2 bytes gives us 4 chars with 4 leftover bits
// - 3 bytes gives us 5 chars with 1 leftover bit
// - 4 bytes gives us 7 chars with 3 leftover bits
// - 5 bytes gives us 8 chars with 0 leftover bits (this is where the cycle repeats)
//
// So really the only way we can get 5-7 leftover bits is if you took a 0, 2 or 5 char output (or
// any 8n + {0,2,5} char output) and added a base32z character to the end. If you do that,
// well, too bad: you're giving invalid output and so we're just going to pretend that extra
// character you added isn't there by not doing anything here.
}
/// Convert a base32z sequence into a std::string of bytes. Undefined behaviour if any characters
/// are not valid (case-insensitive) base32z characters.
template <typename It>
std::string from_base32z(It begin, It end) {
std::string bytes;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>)
bytes.reserve((std::distance(begin, end)*5 + 7) / 8); // == chars*5/8, rounded up.
from_base32z(begin, end, std::back_inserter(bytes));
return bytes;
}
/// Converts base32z digits from a std::string-like object into a std::string of bytes. Undefined
/// behaviour if any characters are not valid (case-insensitive) base32z characters.
template <typename CharT>
std::string from_base32z(std::basic_string_view<CharT> s) { return from_base32z(s.begin(), s.end()); }
inline std::string from_base32z(std::string_view s) { return from_base32z<>(s); }
}
namespace lokimq = oxenmq;

219
lokimq/base64.h
View File

@ -1,219 +1,4 @@
// Copyright (c) 2019-2020, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <string>
#include <string_view>
#include <array>
#include <iterator>
#include <cassert>
#include "../oxenmq/base64.h"
namespace lokimq {
namespace detail {
/// Compile-time generated lookup tables for base64 conversion.
struct b64_table {
// Store the 0-63 decoded value of every possible char; all the chars that aren't valid are set
// to 0. (If you don't trust your data, check it with is_base64 first, which uses these 0's
// to detect invalid characters -- which is why we want a full 256 element array).
char from_b64_lut[256];
// Store the encoded character of every 0-63 (6 bit) value.
char to_b64_lut[64];
// constexpr constructor that fills out the above (and should do it at compile time for any half
// decent compiler).
constexpr b64_table() noexcept : from_b64_lut{}, to_b64_lut{} {
for (unsigned char c = 0; c < 26; c++) {
from_b64_lut[(unsigned char)('A' + c)] = 0 + c;
to_b64_lut[ (unsigned char)( 0 + c)] = 'A' + c;
}
for (unsigned char c = 0; c < 26; c++) {
from_b64_lut[(unsigned char)('a' + c)] = 26 + c;
to_b64_lut[ (unsigned char)(26 + c)] = 'a' + c;
}
for (unsigned char c = 0; c < 10; c++) {
from_b64_lut[(unsigned char)('0' + c)] = 52 + c;
to_b64_lut[ (unsigned char)(52 + c)] = '0' + c;
}
to_b64_lut[62] = '+'; from_b64_lut[(unsigned char) '+'] = 62;
to_b64_lut[63] = '/'; from_b64_lut[(unsigned char) '/'] = 63;
}
// Convert a b64 encoded character into a 0-63 value
constexpr char from_b64(unsigned char c) const noexcept { return from_b64_lut[c]; }
// Convert a 0-31 value into a b64 encoded character
constexpr char to_b64(unsigned char b) const noexcept { return to_b64_lut[b]; }
} constexpr b64_lut;
// This main point of this static assert is to force the compiler to compile-time build the constexpr tables.
static_assert(b64_lut.from_b64('/') == 63 && b64_lut.from_b64('7') == 59 && b64_lut.to_b64(38) == 'm', "");
} // namespace detail
/// Converts bytes into a base64 encoded character sequence.
template <typename InputIt, typename OutputIt>
void to_base64(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "to_base64 requires chars/bytes");
int bits = 0; // Tracks the number of unconsumed bits held in r, will always be in {0, 2, 4}
std::uint_fast16_t r = 0;
while (begin != end) {
r = r << 8 | static_cast<unsigned char>(*begin++);
// we just added 8 bits, so we can *always* consume 6 to produce one character, so (net) we
// are adding 2 bits.
bits += 2;
*out++ = detail::b64_lut.to_b64(r >> bits); // Right-shift off the bits we aren't consuming right now
// Drop the bits we don't want to keep (because we just consumed them)
r &= (1 << bits) - 1;
if (bits == 6) { // We have enough bits to produce a second character (which means we had 4 before and added 8)
bits = 0;
*out++ = detail::b64_lut.to_b64(r);
r = 0;
}
}
// If bits == 0 then we ended our 6-bit outputs coinciding with 8-bit values, i.e. at a multiple
// of 24 bits: this means we don't have anything else to output and don't need any padding.
if (bits == 2) {
// We finished with 2 unconsumed bits, which means we ended 1 byte past a 24-bit group (e.g.
// 1 byte, 4 bytes, 301 bytes, etc.); since we need to always be a multiple of 4 output
// characters that means we've produced 1: so we right-fill 0s to get the next char, then
// add two padding ='s.
*out++ = detail::b64_lut.to_b64(r << 4);
*out++ = '=';
*out++ = '=';
} else if (bits == 4) {
// 4 bits left means we produced 2 6-bit values from the first 2 bytes of a 3-byte group.
// Fill 0s to get the last one, plus one padding output.
*out++ = detail::b64_lut.to_b64(r << 2);
*out++ = '=';
}
}
/// Creates and returns a base64 string from an iterator pair of a character sequence
template <typename It>
std::string to_base64(It begin, It end) {
std::string base64;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>)
base64.reserve((std::distance(begin, end) + 2) / 3 * 4); // bytes*4/3, rounded up to the next multiple of 4
to_base64(begin, end, std::back_inserter(base64));
return base64;
}
/// Creates a base64 string from an iterable, std::string-like object
template <typename CharT>
std::string to_base64(std::basic_string_view<CharT> s) { return to_base64(s.begin(), s.end()); }
inline std::string to_base64(std::string_view s) { return to_base64<>(s); }
/// Returns true if the range is a base64 encoded value; we allow (but do not require) '=' padding,
/// but only at the end, only 1 or 2, and only if it pads out the total to a multiple of 4.
template <typename It>
constexpr bool is_base64(It begin, It end) {
static_assert(sizeof(decltype(*begin)) == 1, "is_base64 requires chars/bytes");
using std::distance;
using std::prev;
// Allow 1 or 2 padding chars *if* they pad it to a multiple of 4.
if (begin != end && distance(begin, end) % 4 == 0) {
auto last = prev(end);
if (static_cast<unsigned char>(*last) == '=')
end = last--;
if (static_cast<unsigned char>(*last) == '=')
end = last;
}
for (; begin != end; ++begin) {
auto c = static_cast<unsigned char>(*begin);
if (detail::b64_lut.from_b64(c) == 0 && c != 'A')
return false;
}
return true;
}
/// Returns true if the string-like value is a base64 encoded value
template <typename CharT>
constexpr bool is_base64(std::basic_string_view<CharT> s) { return is_base64(s.begin(), s.end()); }
constexpr bool is_base64(std::string_view s) { return is_base64(s.begin(), s.end()); }
/// Converts a sequence of base64 digits to bytes. Undefined behaviour if any characters are not
/// valid base64 alphabet characters. It is permitted for the input and output ranges to overlap as
/// long as `out` is no earlier than `begin`. Trailing padding characters are permitted but not
/// required.
///
/// It is possible to provide "impossible" base64 encoded values; for example "YWJja" which has 30
/// bits of data even though a base64 encoded byte string should have 24 (4 chars) or 36 (6 chars)
/// bits for a 3- and 4-byte input, respectively. We ignore any such "impossible" bits, and
/// similarly ignore impossible bits in the bit "overhang"; that means "YWJjZA==" (the proper
/// encoding of "abcd") and "YWJjZB", "YWJjZC", ..., "YWJjZP" all decode to the same "abcd" value:
/// the last 4 bits of the last character are essentially considered padding.
template <typename InputIt, typename OutputIt>
void from_base64(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "from_base64 requires chars/bytes");
uint_fast16_t curr = 0;
int bits = 0; // number of bits we've loaded into val; we always keep this < 8.
while (begin != end) {
auto c = static_cast<unsigned char>(*begin++);
// padding; don't bother checking if we're at the end because is_base64 is a precondition
// and we're allowed UB if it isn't satisfied.
if (c == '=') continue;
curr = curr << 6 | detail::b64_lut.from_b64(c);
if (bits == 0)
bits = 6;
else {
bits -= 2; // Added 6, removing 8
*out++ = static_cast<uint8_t>(curr >> bits);
curr &= (1 << bits) - 1;
}
}
// Don't worry about leftover bits because either they have to be 0, or they can't happen at
// all. See base32z.h for why: the reasoning is exactly the same (except using 6 bits per
// character here instead of 5).
}
/// Converts base64 digits from a iterator pair of characters into a std::string of bytes.
/// Undefined behaviour if any characters are not valid base64 characters.
template <typename It>
std::string from_base64(It begin, It end) {
std::string bytes;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>)
bytes.reserve(std::distance(begin, end)*6 / 8); // each digit carries 6 bits; this may overallocate by 1-2 bytes due to padding
from_base64(begin, end, std::back_inserter(bytes));
return bytes;
}
/// Converts base64 digits from a std::string-like object into a std::string of bytes. Undefined
/// behaviour if any characters are not valid base64 characters.
template <typename CharT>
std::string from_base64(std::basic_string_view<CharT> s) { return from_base64(s.begin(), s.end()); }
inline std::string from_base64(std::string_view s) { return from_base64<>(s); }
}
namespace lokimq = oxenmq;

279
lokimq/batch.h
View File

@ -1,279 +1,4 @@
// Copyright (c) 2020, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <exception>
#include <functional>
#include <vector>
#include "lokimq.h"
#include "../oxenmq/batch.h"
namespace lokimq {
namespace detail {
enum class BatchState {
running, // there are still jobs to run (or running)
complete, // the batch is complete but still has a completion job to call
done // the batch is complete and has no completion function
};
struct BatchStatus {
BatchState state;
int thread;
};
// Virtual base class for Batch<R>
class Batch {
public:
// Returns the number of jobs in this batch and whether any of them are thread-specific
virtual std::pair<size_t, bool> size() const = 0;
// Returns a vector of exactly the same length of size().first containing the tagged thread ids
// of the batch jobs or 0 for general jobs.
virtual std::vector<int> threads() const = 0;
// Called in a worker thread to run the job
virtual void run_job(int i) = 0;
// Called in the main proxy thread when the worker returns from finishing a job. The return
// value tells us whether the current finishing job finishes off the batch: `running` to tell us
// there are more jobs; `complete` to tell us that the jobs are done but the completion function
// needs to be called; and `done` to signal that the jobs are done and there is no completion
// function.
virtual BatchStatus job_finished() = 0;
// Called by a worker; not scheduled until all jobs are done.
virtual void job_completion() = 0;
virtual ~Batch() = default;
};
}
/**
* Simple class that can either hold a result or an exception and retrieves the result (or raises
* the exception) via a .get() method.
*
* This is designed to be like a very stripped down version of a std::promise/std::future pair. We
* reimplemented it, however, because by ditching all the thread synchronization that promise/future
* guarantees we can substantially reduce call overhead (by a factor of ~8 according to benchmarking
* code). Since LokiMQ's proxy<->worker communication channel already gives us thread that overhead
* would just be wasted.
*
* @tparam R the value type held by the result; must be default constructible. Note, however, that
* there are specializations provided for lvalue references types and `void` (which obviously don't
* satisfy this).
*/
template <typename R, typename SFINAE = void>
class job_result {
R value;
std::exception_ptr exc;
public:
/// Sets the value. Should be called only once, or not at all if set_exception was called.
void set_value(R&& v) { value = std::move(v); }
/// Sets the exception, which will be rethrown when `get()` is called. Should be called
/// only once, or not at all if set_value() was called.
void set_exception(std::exception_ptr e) { exc = std::move(e); }
/// Retrieves the value. If an exception was set instead of a value then that exception is
/// thrown instead. Note that the interval value is moved out of the held value so you should
/// not call this multiple times.
R get() {
if (exc) std::rethrow_exception(exc);
return std::move(value);
}
};
/** job_result specialization for reference types */
template <typename R>
class job_result<R, std::enable_if_t<std::is_lvalue_reference<R>::value>> {
std::remove_reference_t<R>* value_ptr;
std::exception_ptr exc;
public:
void set_value(R v) { value_ptr = &v; }
void set_exception(std::exception_ptr e) { exc = std::move(e); }
R get() {
if (exc) std::rethrow_exception(exc);
return *value_ptr;
}
};
/** job_result specialization for void; there is no value, but exceptions are still captured
* (rethrown when `get()` is called).
*/
template<>
class job_result<void> {
std::exception_ptr exc;
public:
void set_exception(std::exception_ptr e) { exc = std::move(e); }
// Returns nothing, but rethrows if there is a captured exception.
void get() { if (exc) std::rethrow_exception(exc); }
};
/// Helper class used to set up batches of jobs to be scheduled via the lokimq job handler.
///
/// @tparam R - the return type of the individual jobs
///
template <typename R>
class Batch final : private detail::Batch {
friend class LokiMQ;
public:
/// The completion function type, called after all jobs have finished.
using CompletionFunc = std::function<void(std::vector<job_result<R>> results)>;
// Default constructor
Batch() = default;
// movable
Batch(Batch&&) = default;
Batch &operator=(Batch&&) = default;
// non-copyable
Batch(const Batch&) = delete;
Batch &operator=(const Batch&) = delete;
private:
std::vector<std::pair<std::function<R()>, int>> jobs;
std::vector<job_result<R>> results;
CompletionFunc complete;
std::size_t jobs_outstanding = 0;
int complete_in_thread = 0;
bool started = false;
bool tagged_thread_jobs = false;
void check_not_started() {
if (started)
throw std::logic_error("Cannot add jobs or completion function after starting a lokimq::Batch!");
}
public:
/// Preallocates space in the internal vector that stores jobs.
void reserve(std::size_t num) {
jobs.reserve(num);
results.reserve(num);
}
/// Adds a job. This takes any callable object that is invoked with no arguments and returns R
/// (the Batch return type). The tasks will be scheduled and run when the next worker thread is
/// available. The called function may throw exceptions (which will be propagated to the
/// completion function through the job_result values). There is no guarantee on the order of
/// invocation of the jobs.
///
/// \param job the callback
/// \param thread an optional TaggedThreadID indicating a thread in which this job must run
void add_job(std::function<R()> job, std::optional<TaggedThreadID> thread = std::nullopt) {
check_not_started();
if (thread && thread->_id == -1)
// There are some special case internal jobs where we allow this, but they use the
// private method below that doesn't have this check.
throw std::logic_error{"Cannot add a proxy thread batch job -- this makes no sense"};
add_job(std::move(job), thread ? thread->_id : 0);
}
/// Sets the completion function to invoke after all jobs have finished. If this is not set
/// then jobs simply run and results are discarded.
///
/// \param comp - function to call when all jobs have finished
/// \param thread - optional tagged thread in which to schedule the completion job. If not
/// provided then the completion job is scheduled in the pool of batch job threads.
///
/// `thread` can be provided the value &LokiMQ::run_in_proxy to invoke the completion function
/// *IN THE PROXY THREAD* itself after all jobs have finished. Be very, very careful: this
/// should be a nearly trivial job that does not require any substantial CPU time and does not
/// block for any reason. This is only intended for the case where the completion job is so
/// trivial that it will take less time than simply queuing the job to be executed by another
/// thread.
void completion(CompletionFunc comp, std::optional<TaggedThreadID> thread = std::nullopt) {
check_not_started();
if (complete)
throw std::logic_error("Completion function can only be set once");
complete = std::move(comp);
complete_in_thread = thread ? thread->_id : 0;
}
private:
void add_job(std::function<R()> job, int thread_id) {
jobs.emplace_back(std::move(job), thread_id);
results.emplace_back();
jobs_outstanding++;
if (thread_id != 0)
tagged_thread_jobs = true;
}
std::pair<std::size_t, bool> size() const override {
return {jobs.size(), tagged_thread_jobs};
}
std::vector<int> threads() const override {
std::vector<int> t;
t.reserve(jobs.size());
for (auto& j : jobs)
t.push_back(j.second);
return t;
};
template <typename S = R>
void set_value(job_result<S>& r, std::function<S()>& f) { r.set_value(f()); }
void set_value(job_result<void>&, std::function<void()>& f) { f(); }
void run_job(const int i) override {
// called by worker thread
auto& r = results[i];
try {
set_value(r, jobs[i].first);
} catch (...) {
r.set_exception(std::current_exception());
}
}
detail::BatchStatus job_finished() override {
--jobs_outstanding;
if (jobs_outstanding)
return {detail::BatchState::running, 0};
if (complete)
return {detail::BatchState::complete, complete_in_thread};
return {detail::BatchState::done, 0};
}
void job_completion() override {
return complete(std::move(results));
}
};
template <typename R>
void LokiMQ::batch(Batch<R>&& batch) {
if (batch.size().first == 0)
throw std::logic_error("Cannot batch a a job batch with 0 jobs");
// Need to send this over to the proxy thread via the base class pointer. It assumes ownership.
auto* baseptr = static_cast<detail::Batch*>(new Batch<R>(std::move(batch)));
detail::send_control(get_control_socket(), "BATCH", bt_serialize(reinterpret_cast<uintptr_t>(baseptr)));
}
}
namespace lokimq = oxenmq;

915
lokimq/bt_serialize.h
View File

@ -1,915 +1,4 @@
// Copyright (c) 2019-2020, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include "../oxenmq/bt_serialize.h"
#include <vector>
#include <functional>
#include <cstring>
#include <sstream>
#include <ostream>
#include <string>
#include <string_view>
#include "variant.h"
#include <cstdint>
#include <limits>
#include <stdexcept>
#include <type_traits>
#include <utility>
#include <tuple>
#include <algorithm>
#include "bt_value.h"
namespace lokimq {
using namespace std::literals;
/** \file
* LokiMQ serialization for internal commands is very simple: we support two primitive types,
* strings and integers, and two container types, lists and dicts with string keys. On the wire
* these go in BitTorrent byte encoding as described in BEP-0003
* (https://www.bittorrent.org/beps/bep_0003.html#bencoding).
*
* On the C++ side, on input we allow strings, integral types, STL-like containers of these types,
* and STL-like containers of pairs with a string first value and any of these types as second
* value. We also accept std::variants of these.
*
* One minor deviation from BEP-0003 is that we don't support serializing values that don't fit in a
* 64-bit integer (BEP-0003 specifies arbitrary precision integers).
*
* On deserialization we can either deserialize into a special bt_value type supports everything
* (with arbitrary nesting), or we can fill a container of your given type (though this fails if the
* container isn't compatible with the deserialized data).
*
* There is also a stream deserialization that allows you to deserialize without needing heap
* allocations (as long as you know the precise data structure layout).
*/
/// Exception throw if deserialization fails
class bt_deserialize_invalid : public std::invalid_argument {
using std::invalid_argument::invalid_argument;
};
/// A more specific subclass that is thown if the serialization type is an initial mismatch: for
/// example, trying deserializing an int but the next thing in input is a list. This is not,
/// however, thrown if the type initially looks fine but, say, a nested serialization fails. This
/// error will only be thrown when the input stream has not been advanced (and so can be tried for a
/// different type).
class bt_deserialize_invalid_type : public bt_deserialize_invalid {
using bt_deserialize_invalid::bt_deserialize_invalid;
};
namespace detail {
/// Reads digits into an unsigned 64-bit int.
uint64_t extract_unsigned(std::string_view& s);
// (Provide non-constant lvalue and rvalue ref functions so that we only accept explicit
// string_views but not implicitly converted ones)
inline uint64_t extract_unsigned(std::string_view&& s) { return extract_unsigned(s); }
// Fallback base case; we only get here if none of the partial specializations below work
template <typename T, typename SFINAE = void>
struct bt_serialize { static_assert(!std::is_same_v<T, T>, "Cannot serialize T: unsupported type for bt serialization"); };
template <typename T, typename SFINAE = void>
struct bt_deserialize { static_assert(!std::is_same_v<T, T>, "Cannot deserialize T: unsupported type for bt deserialization"); };
/// Checks that we aren't at the end of a string view and throws if we are.
inline void bt_need_more(const std::string_view &s) {
if (s.empty())
throw bt_deserialize_invalid{"Unexpected end of string while deserializing"};
}
/// Deserializes a signed or unsigned 64-bit integer from a string. Sets the second bool to true
/// iff the value read was negative, false if positive; in either case the unsigned value is return
/// in .first. Throws an exception if the read value doesn't fit in a int64_t (if negative) or a
/// uint64_t (if positive). Removes consumed characters from the string_view.
std::pair<uint64_t, bool> bt_deserialize_integer(std::string_view& s);
/// Integer specializations
template <typename T>
struct bt_serialize<T, std::enable_if_t<std::is_integral_v<T>>> {
static_assert(sizeof(T) <= sizeof(uint64_t), "Serialization of integers larger than uint64_t is not supported");
void operator()(std::ostream &os, const T &val) {
// Cast 1-byte types to a larger type to avoid iostream interpreting them as single characters
using output_type = std::conditional_t<(sizeof(T) > 1), T, std::conditional_t<std::is_signed_v<T>, int, unsigned>>;
os << 'i' << static_cast<output_type>(val) << 'e';
}
};
template <typename T>
struct bt_deserialize<T, std::enable_if_t<std::is_integral_v<T>>> {
void operator()(std::string_view& s, T &val) {
constexpr uint64_t umax = static_cast<uint64_t>(std::numeric_limits<T>::max());
constexpr int64_t smin = static_cast<int64_t>(std::numeric_limits<T>::min());
auto [magnitude, negative] = bt_deserialize_integer(s);
if (std::is_signed_v<T>) {
if (!negative) {
if (magnitude > umax)
throw bt_deserialize_invalid("Integer deserialization failed: found too-large value " + std::to_string(magnitude) + " > " + std::to_string(umax));
val = static_cast<T>(magnitude);
} else {
auto sval = -static_cast<int64_t>(magnitude);
if (!std::is_same_v<T, int64_t> && sval < smin)
throw bt_deserialize_invalid("Integer deserialization failed: found too-low value " + std::to_string(sval) + " < " + std::to_string(smin));
val = static_cast<T>(sval);
}
} else {
if (negative)
throw bt_deserialize_invalid("Integer deserialization failed: found negative value -" + std::to_string(magnitude) + " but type is unsigned");
if (!std::is_same_v<T, uint64_t> && magnitude > umax)
throw bt_deserialize_invalid("Integer deserialization failed: found too-large value " + std::to_string(magnitude) + " > " + std::to_string(umax));
val = static_cast<T>(magnitude);
}
}
};
extern template struct bt_deserialize<int64_t>;
extern template struct bt_deserialize<uint64_t>;
template <>
struct bt_serialize<std::string_view> {
void operator()(std::ostream &os, const std::string_view &val) { os << val.size(); os.put(':'); os.write(val.data(), val.size()); }
};
template <>
struct bt_deserialize<std::string_view> {
void operator()(std::string_view& s, std::string_view& val);
};
/// String specialization
template <>
struct bt_serialize<std::string> {
void operator()(std::ostream &os, const std::string &val) { bt_serialize<std::string_view>{}(os, val); }
};
template <>
struct bt_deserialize<std::string> {
void operator()(std::string_view& s, std::string& val) { std::string_view view; bt_deserialize<std::string_view>{}(s, view); val = {view.data(), view.size()}; }
};
/// char * and string literals -- we allow serialization for convenience, but not deserialization
template <>
struct bt_serialize<char *> {
void operator()(std::ostream &os, const char *str) { bt_serialize<std::string_view>{}(os, {str, std::strlen(str)}); }
};
template <size_t N>
struct bt_serialize<char[N]> {
void operator()(std::ostream &os, const char *str) { bt_serialize<std::string_view>{}(os, {str, N-1}); }
};
/// Partial dict validity; we don't check the second type for serializability, that will be handled
/// via the base case static_assert if invalid.
template <typename T, typename = void> struct is_bt_input_dict_container_impl : std::false_type {};
template <typename T>
struct is_bt_input_dict_container_impl<T, std::enable_if_t<
std::is_same_v<std::string, std::remove_cv_t<typename T::value_type::first_type>> ||
std::is_same_v<std::string_view, std::remove_cv_t<typename T::value_type::first_type>>,
std::void_t<typename T::const_iterator /* is const iterable */,
typename T::value_type::second_type /* has a second type */>>>
: std::true_type {};
/// Determines whether the type looks like something we can insert into (using `v.insert(v.end(), x)`)
template <typename T, typename = void> struct is_bt_insertable_impl : std::false_type {};
template <typename T>
struct is_bt_insertable_impl<T,
std::void_t<decltype(std::declval<T>().insert(std::declval<T>().end(), std::declval<typename T::value_type>()))>>
: std::true_type {};
template <typename T>
constexpr bool is_bt_insertable = is_bt_insertable_impl<T>::value;
/// Determines whether the given type looks like a compatible map (i.e. has std::string keys) that
/// we can insert into.
template <typename T, typename = void> struct is_bt_output_dict_container_impl : std::false_type {};
template <typename T>
struct is_bt_output_dict_container_impl<T, std::enable_if_t<
std::is_same_v<std::string, std::remove_cv_t<typename T::value_type::first_type>> && is_bt_insertable<T>,
std::void_t<typename T::value_type::second_type /* has a second type */>>>
: std::true_type {};
template <typename T>
constexpr bool is_bt_output_dict_container = is_bt_output_dict_container_impl<T>::value;
template <typename T>
constexpr bool is_bt_input_dict_container = is_bt_output_dict_container_impl<T>::value;
// Sanity checks:
static_assert(is_bt_input_dict_container<bt_dict>);
static_assert(is_bt_output_dict_container<bt_dict>);
/// Specialization for a dict-like container (such as an unordered_map). We accept anything for a
/// dict that is const iterable over something that looks like a pair with std::string for first
/// value type. The value (i.e. second element of the pair) also must be serializable.
template <typename T>
struct bt_serialize<T, std::enable_if_t<is_bt_input_dict_container<T>>> {
using second_type = typename T::value_type::second_type;
using ref_pair = std::reference_wrapper<const typename T::value_type>;
void operator()(std::ostream &os, const T &dict) {
os << 'd';
std::vector<ref_pair> pairs;
pairs.reserve(dict.size());
for (const auto &pair : dict)
pairs.emplace(pairs.end(), pair);
std::sort(pairs.begin(), pairs.end(), [](ref_pair a, ref_pair b) { return a.get().first < b.get().first; });
for (auto &ref : pairs) {
bt_serialize<std::string>{}(os, ref.get().first);
bt_serialize<second_type>{}(os, ref.get().second);
}
os << 'e';
}
};
template <typename T>
struct bt_deserialize<T, std::enable_if_t<is_bt_output_dict_container<T>>> {
using second_type = typename T::value_type::second_type;
void operator()(std::string_view& s, T& dict) {
// Smallest dict is 2 bytes "de", for an empty dict.
if (s.size() < 2) throw bt_deserialize_invalid("Deserialization failed: end of string found where dict expected");
if (s[0] != 'd') throw bt_deserialize_invalid_type("Deserialization failed: expected 'd', found '"s + s[0] + "'"s);
s.remove_prefix(1);
dict.clear();
bt_deserialize<std::string> key_deserializer;
bt_deserialize<second_type> val_deserializer;
while (!s.empty() && s[0] != 'e') {
std::string key;
second_type val;
key_deserializer(s, key);
val_deserializer(s, val);
dict.insert(dict.end(), typename T::value_type{std::move(key), std::move(val)});
}
if (s.empty())
throw bt_deserialize_invalid("Deserialization failed: encountered end of string before dict was finished");
s.remove_prefix(1); // Consume the 'e'
}
};
/// Accept anything that looks iterable; value serialization validity isn't checked here (it fails
/// via the base case static assert).
template <typename T, typename = void> struct is_bt_input_list_container_impl : std::false_type {};
template <typename T>
struct is_bt_input_list_container_impl<T, std::enable_if_t<
!std::is_same_v<T, std::string> && !std::is_same_v<T, std::string_view> && !is_bt_input_dict_container<T>,
std::void_t<typename T::const_iterator, typename T::value_type>>>
: std::true_type {};
template <typename T, typename = void> struct is_bt_output_list_container_impl : std::false_type {};
template <typename T>
struct is_bt_output_list_container_impl<T, std::enable_if_t<
!std::is_same_v<T, std::string> && !is_bt_output_dict_container<T> && is_bt_insertable<T>>>
: std::true_type {};
template <typename T>
constexpr bool is_bt_output_list_container = is_bt_output_list_container_impl<T>::value;
template <typename T>
constexpr bool is_bt_input_list_container = is_bt_input_list_container_impl<T>::value;
// Sanity checks:
static_assert(is_bt_input_list_container<bt_list>);
static_assert(is_bt_output_list_container<bt_list>);
/// List specialization
template <typename T>
struct bt_serialize<T, std::enable_if_t<is_bt_input_list_container<T>>> {
void operator()(std::ostream& os, const T& list) {
os << 'l';
for (const auto &v : list)
bt_serialize<std::remove_cv_t<typename T::value_type>>{}(os, v);
os << 'e';
}
};
template <typename T>
struct bt_deserialize<T, std::enable_if_t<is_bt_output_list_container<T>>> {
using value_type = typename T::value_type;
void operator()(std::string_view& s, T& list) {
// Smallest list is 2 bytes "le", for an empty list.
if (s.size() < 2) throw bt_deserialize_invalid("Deserialization failed: end of string found where list expected");
if (s[0] != 'l') throw bt_deserialize_invalid_type("Deserialization failed: expected 'l', found '"s + s[0] + "'"s);
s.remove_prefix(1);
list.clear();
bt_deserialize<value_type> deserializer;
while (!s.empty() && s[0] != 'e') {
value_type v;
deserializer(s, v);
list.insert(list.end(), std::move(v));
}
if (s.empty())
throw bt_deserialize_invalid("Deserialization failed: encountered end of string before list was finished");
s.remove_prefix(1); // Consume the 'e'
}
};
/// Serializes a tuple or pair of serializable values (as a list on the wire)
/// Common implementation for both tuple and pair:
template <template<typename...> typename Tuple, typename... T>
struct bt_serialize_tuple {
private:
template <size_t... Is>
void operator()(std::ostream& os, const Tuple<T...>& elems, std::index_sequence<Is...>) {
os << 'l';
(bt_serialize<T>{}(os, std::get<Is>(elems)), ...);
os << 'e';
}
public:
void operator()(std::ostream& os, const Tuple<T...>& elems) {
operator()(os, elems, std::index_sequence_for<T...>{});
}
};
template <template<typename...> typename Tuple, typename... T>
struct bt_deserialize_tuple {
private:
template <size_t... Is>
void operator()(std::string_view& s, Tuple<T...>& elems, std::index_sequence<Is...>) {
// Smallest list is 2 bytes "le", for an empty list.
if (s.size() < 2) throw bt_deserialize_invalid("Deserialization failed: end of string found where tuple expected");
if (s[0] != 'l') throw bt_deserialize_invalid_type("Deserialization of tuple failed: expected 'l', found '"s + s[0] + "'"s);
s.remove_prefix(1);
(bt_deserialize<T>{}(s, std::get<Is>(elems)), ...);
if (s.empty())
throw bt_deserialize_invalid("Deserialization failed: encountered end of string before tuple was finished");
if (s[0] != 'e')
throw bt_deserialize_invalid("Deserialization failed: expected end of tuple but found something else");
s.remove_prefix(1); // Consume the 'e'
}
public:
void operator()(std::string_view& s, Tuple<T...>& elems) {
operator()(s, elems, std::index_sequence_for<T...>{});
}
};
template <typename... T>
struct bt_serialize<std::tuple<T...>> : bt_serialize_tuple<std::tuple, T...> {};
template <typename... T>
struct bt_deserialize<std::tuple<T...>> : bt_deserialize_tuple<std::tuple, T...> {};
template <typename S, typename T>
struct bt_serialize<std::pair<S, T>> : bt_serialize_tuple<std::pair, S, T> {};
template <typename S, typename T>
struct bt_deserialize<std::pair<S, T>> : bt_deserialize_tuple<std::pair, S, T> {};
template <typename T>
constexpr bool is_bt_tuple = false;
template <typename... T>
constexpr bool is_bt_tuple<std::tuple<T...>> = true;
template <typename S, typename T>
constexpr bool is_bt_tuple<std::pair<S, T>> = true;
template <typename T>
constexpr bool is_bt_deserializable = std::is_same_v<T, std::string> || std::is_integral_v<T> ||
is_bt_output_dict_container<T> || is_bt_output_list_container<T> || is_bt_tuple<T>;
// General template and base case; this base will only actually be invoked when Ts... is empty,
// which means we reached the end without finding any variant type capable of holding the value.
template <typename SFINAE, typename Variant, typename... Ts>
struct bt_deserialize_try_variant_impl {
void operator()(std::string_view&, Variant&) {
throw bt_deserialize_invalid("Deserialization failed: could not deserialize value into any variant type");
}
};
template <typename... Ts, typename Variant>
void bt_deserialize_try_variant(std::string_view& s, Variant& variant) {
bt_deserialize_try_variant_impl<void, Variant, Ts...>{}(s, variant);
}
template <typename Variant, typename T, typename... Ts>
struct bt_deserialize_try_variant_impl<std::enable_if_t<is_bt_deserializable<T>>, Variant, T, Ts...> {
void operator()(std::string_view& s, Variant& variant) {
if ( is_bt_output_list_container<T> ? s[0] == 'l' :
is_bt_tuple<T> ? s[0] == 'l' :
is_bt_output_dict_container<T> ? s[0] == 'd' :
std::is_integral_v<T> ? s[0] == 'i' :
std::is_same_v<T, std::string> ? s[0] >= '0' && s[0] <= '9' :
false) {
T val;
bt_deserialize<T>{}(s, val);
variant = std::move(val);
} else {
bt_deserialize_try_variant<Ts...>(s, variant);
}
}
};
template <typename Variant, typename T, typename... Ts>
struct bt_deserialize_try_variant_impl<std::enable_if_t<!is_bt_deserializable<T>>, Variant, T, Ts...> {
void operator()(std::string_view& s, Variant& variant) {
// Unsupported deserialization type, skip it
bt_deserialize_try_variant<Ts...>(s, variant);
}
};
// Serialization of a variant; all variant types must be bt-serializable.
template <typename... Ts>
struct bt_serialize<std::variant<Ts...>, std::void_t<bt_serialize<Ts>...>> {
void operator()(std::ostream& os, const std::variant<Ts...>& val) {
var::visit(
[&os] (const auto& val) {
using T = std::remove_cv_t<std::remove_reference_t<decltype(val)>>;
bt_serialize<T>{}(os, val);
},
val);
}
};
// Deserialization to a variant; at least one variant type must be bt-deserializble.
template <typename... Ts>
struct bt_deserialize<std::variant<Ts...>, std::enable_if_t<(is_bt_deserializable<Ts> || ...)>> {
void operator()(std::string_view& s, std::variant<Ts...>& val) {
bt_deserialize_try_variant<Ts...>(s, val);
}
};
template <>
struct bt_serialize<bt_value> : bt_serialize<bt_variant> {};
template <>
struct bt_deserialize<bt_value> {
void operator()(std::string_view& s, bt_value& val);
};
template <typename T>
struct bt_stream_serializer {
const T &val;
explicit bt_stream_serializer(const T &val) : val{val} {}
operator std::string() const {
std::ostringstream oss;
oss << *this;
return oss.str();
}
};
template <typename T>
std::ostream &operator<<(std::ostream &os, const bt_stream_serializer<T> &s) {
bt_serialize<T>{}(os, s.val);
return os;
}
} // namespace detail
/// Returns a wrapper around a value reference that can serialize the value directly to an output
/// stream. This class is intended to be used inline (i.e. without being stored) as in:
///
/// std::list<int> my_list{{1,2,3}};
/// std::cout << bt_serializer(my_list);
///
/// While it is possible to store the returned object and use it, such as:
///
/// auto encoded = bt_serializer(42);
/// std::cout << encoded;
///
/// this approach is not generally recommended: the returned object stores a reference to the
/// passed-in type, which may not survive. If doing this note that it is the caller's
/// responsibility to ensure the serializer is not used past the end of the lifetime of the value
/// being serialized.
///
/// Also note that serializing directly to an output stream is more efficient as no intermediate
/// string containing the entire serialization has to be constructed.
///
template <typename T>
detail::bt_stream_serializer<T> bt_serializer(const T &val) { return detail::bt_stream_serializer<T>{val}; }
/// Serializes the given value into a std::string.
///
/// int number = 42;
/// std::string encoded = bt_serialize(number);
/// // Equivalent:
/// //auto encoded = (std::string) bt_serialize(number);
///
/// This takes any serializable type: integral types, strings, lists of serializable types, and
/// string->value maps of serializable types.
template <typename T>
std::string bt_serialize(const T &val) { return bt_serializer(val); }
/// Deserializes the given string view directly into `val`. Usage:
///
/// std::string encoded = "i42e";
/// int value;
/// bt_deserialize(encoded, value); // Sets value to 42
///
template <typename T, std::enable_if_t<!std::is_const_v<T>, int> = 0>
void bt_deserialize(std::string_view s, T& val) {
return detail::bt_deserialize<T>{}(s, val);
}
/// Deserializes the given string_view into a `T`, which is returned.
///
/// std::string encoded = "li1ei2ei3ee"; // bt-encoded list of ints: [1,2,3]
/// auto mylist = bt_deserialize<std::list<int>>(encoded);
///
template <typename T>
T bt_deserialize(std::string_view s) {
T val;
bt_deserialize(s, val);
return val;
}
/// Deserializes the given value into a generic `bt_value` type (wrapped std::variant) which is
/// capable of holding all possible BT-encoded values (including recursion).
///
/// Example:
///
/// std::string encoded = "i42e";
/// auto val = bt_get(encoded);
/// int v = get_int<int>(val); // fails unless the encoded value was actually an integer that
/// // fits into an `int`
///
inline bt_value bt_get(std::string_view s) {
return bt_deserialize<bt_value>(s);
}
/// Helper functions to extract a value of some integral type from a bt_value which contains either
/// a int64_t or uint64_t. Does range checking, throwing std::overflow_error if the stored value is
/// outside the range of the target type.
///
/// Example:
///
/// std::string encoded = "i123456789e";
/// auto val = bt_get(encoded);
/// auto v = get_int<uint32_t>(val); // throws if the decoded value doesn't fit in a uint32_t
template <typename IntType, std::enable_if_t<std::is_integral_v<IntType>, int> = 0>
IntType get_int(const bt_value &v) {
if (auto* value = std::get_if<uint64_t>(&v)) {
if constexpr (!std::is_same_v<IntType, uint64_t>)
if (*value > static_cast<uint64_t>(std::numeric_limits<IntType>::max()))
throw std::overflow_error("Unable to extract integer value: stored value is too large for the requested type");
return static_cast<IntType>(*value);
}
int64_t value = var::get<int64_t>(v); // throws if no int contained
if constexpr (!std::is_same_v<IntType, int64_t>)
if (value > static_cast<int64_t>(std::numeric_limits<IntType>::max())
|| value < static_cast<int64_t>(std::numeric_limits<IntType>::min()))
throw std::overflow_error("Unable to extract integer value: stored value is outside the range of the requested type");
return static_cast<IntType>(value);
}
namespace detail {
template <typename Tuple, size_t... Is>
void get_tuple_impl(Tuple& t, const bt_list& l, std::index_sequence<Is...>);
}
/// Converts a bt_list into the given template std::tuple or std::pair. Throws a
/// std::invalid_argument if the list has the wrong size or wrong element types. Supports recursion
/// (i.e. if the tuple itself contains tuples or pairs). The tuple (or nested tuples) may only
/// contain integral types, strings, string_views, bt_list, bt_dict, and tuples/pairs of those.
template <typename Tuple>
Tuple get_tuple(const bt_list& x) {
Tuple t;
detail::get_tuple_impl(t, x, std::make_index_sequence<std::tuple_size_v<Tuple>>{});
return t;
}
template <typename Tuple>
Tuple get_tuple(const bt_value& x) {
return get_tuple<Tuple>(var::get<bt_list>(static_cast<const bt_variant&>(x)));
}
namespace detail {
template <typename T, typename It>
void get_tuple_impl_one(T& t, It& it) {
const bt_variant& v = *it++;
if constexpr (std::is_integral_v<T>) {
t = lokimq::get_int<T>(v);
} else if constexpr (is_bt_tuple<T>) {
if (std::holds_alternative<bt_list>(v))
throw std::invalid_argument{"Unable to convert tuple: cannot create sub-tuple from non-bt_list"};
t = get_tuple<T>(var::get<bt_list>(v));
} else if constexpr (std::is_same_v<std::string, T> || std::is_same_v<std::string_view, T>) {
// If we request a string/string_view, we might have the other one and need to copy/view it.
if (std::holds_alternative<std::string_view>(v))
t = var::get<std::string_view>(v);
else
t = var::get<std::string>(v);
} else {
t = var::get<T>(v);
}
}
template <typename Tuple, size_t... Is>
void get_tuple_impl(Tuple& t, const bt_list& l, std::index_sequence<Is...>) {
if (l.size() != sizeof...(Is))
throw std::invalid_argument{"Unable to convert tuple: bt_list has wrong size"};
auto it = l.begin();
(get_tuple_impl_one(std::get<Is>(t), it), ...);
}
} // namespace detail
/// Class that allows you to walk through a bt-encoded list in memory without copying or allocating
/// memory. It accesses existing memory directly and so the caller must ensure that the referenced
/// memory stays valid for the lifetime of the bt_list_consumer object.
class bt_list_consumer {
protected:
std::string_view data;
bt_list_consumer() = default;
public:
bt_list_consumer(std::string_view data_);
/// Copy constructor. Making a copy copies the current position so can be used for multipass
/// iteration through a list.
bt_list_consumer(const bt_list_consumer&) = default;
bt_list_consumer& operator=(const bt_list_consumer&) = default;
/// Get a copy of the current buffer
std::string_view current_buffer() const { return data; }
/// Returns true if the next value indicates the end of the list
bool is_finished() const { return data.front() == 'e'; }
/// Returns true if the next element looks like an encoded string
bool is_string() const { return data.front() >= '0' && data.front() <= '9'; }
/// Returns true if the next element looks like an encoded integer
bool is_integer() const { return data.front() == 'i'; }
/// Returns true if the next element looks like an encoded negative integer
bool is_negative_integer() const { return is_integer() && data.size() >= 2 && data[1] == '-'; }
/// Returns true if the next element looks like an encoded list
bool is_list() const { return data.front() == 'l'; }
/// Returns true if the next element looks like an encoded dict
bool is_dict() const { return data.front() == 'd'; }
/// Attempt to parse the next value as a string (and advance just past it). Throws if the next
/// value is not a string.
std::string consume_string();
std::string_view consume_string_view();
/// Attempts to parse the next value as an integer (and advance just past it). Throws if the
/// next value is not an integer.
template <typename IntType>
IntType consume_integer() {
if (!is_integer()) throw bt_deserialize_invalid_type{"next value is not an integer"};
std::string_view next{data};
IntType ret;
detail::bt_deserialize<IntType>{}(next, ret);
data = next;
return ret;
}
/// Consumes a list, return it as a list-like type. Can also be used for tuples/pairs. This
/// typically requires dynamic allocation, but only has to parse the data once. Compare with
/// consume_list_data() which allows alloc-free traversal, but requires parsing twice (if the
/// contents are to be used).
template <typename T = bt_list>
T consume_list() {
T list;
consume_list(list);
return list;
}
/// Same as above, but takes a pre-existing list-like data type.
template <typename T>
void consume_list(T& list) {
if (!is_list()) throw bt_deserialize_invalid_type{"next bt value is not a list"};
std::string_view n{data};
detail::bt_deserialize<T>{}(n, list);
data = n;
}
/// Consumes a dict, return it as a dict-like type. This typically requires dynamic allocation,
/// but only has to parse the data once. Compare with consume_dict_data() which allows
/// alloc-free traversal, but requires parsing twice (if the contents are to be used).
template <typename T = bt_dict>
T consume_dict() {
T dict;
consume_dict(dict);
return dict;
}
/// Same as above, but takes a pre-existing dict-like data type.
template <typename T>
void consume_dict(T& dict) {
if (!is_dict()) throw bt_deserialize_invalid_type{"next bt value is not a dict"};
std::string_view n{data};
detail::bt_deserialize<T>{}(n, dict);
data = n;
}
/// Consumes a value without returning it.
void skip_value();
/// Attempts to parse the next value as a list and returns the string_view that contains the
/// entire thing. This is recursive into both lists and dicts and likely to be quite
/// inefficient for large, nested structures (unless the values only need to be skipped but
/// aren't separately needed). This, however, does not require dynamic memory allocation.
std::string_view consume_list_data();
/// Attempts to parse the next value as a dict and returns the string_view that contains the
/// entire thing. This is recursive into both lists and dicts and likely to be quite
/// inefficient for large, nested structures (unless the values only need to be skipped but
/// aren't separately needed). This, however, does not require dynamic memory allocation.
std::string_view consume_dict_data();
};
/// Class that allows you to walk through key-value pairs of a bt-encoded dict in memory without
/// copying or allocating memory. It accesses existing memory directly and so the caller must
/// ensure that the referenced memory stays valid for the lifetime of the bt_dict_consumer object.
class bt_dict_consumer : private bt_list_consumer {
std::string_view key_;
/// Consume the key if not already consumed and there is a key present (rather than 'e').
/// Throws exception if what should be a key isn't a string, or if the key consumes the entire
/// data (i.e. requires that it be followed by something). Returns true if the key was consumed
/// (either now or previously and cached).
bool consume_key();
/// Clears the cached key and returns it. Must have already called consume_key directly or
/// indirectly via one of the `is_{...}` methods.
std::string_view flush_key() {
std::string_view k;
k.swap(key_);
return k;
}
public:
bt_dict_consumer(std::string_view data_);
/// Copy constructor. Making a copy copies the current position so can be used for multipass
/// iteration through a list.
bt_dict_consumer(const bt_dict_consumer&) = default;
bt_dict_consumer& operator=(const bt_dict_consumer&) = default;
/// Returns true if the next value indicates the end of the dict
bool is_finished() { return !consume_key() && data.front() == 'e'; }
/// Operator bool is an alias for `!is_finished()`
operator bool() { return !is_finished(); }
/// Returns true if the next value looks like an encoded string
bool is_string() { return consume_key() && data.front() >= '0' && data.front() <= '9'; }
/// Returns true if the next element looks like an encoded integer
bool is_integer() { return consume_key() && data.front() == 'i'; }
/// Returns true if the next element looks like an encoded negative integer
bool is_negative_integer() { return is_integer() && data.size() >= 2 && data[1] == '-'; }
/// Returns true if the next element looks like an encoded list
bool is_list() { return consume_key() && data.front() == 'l'; }
/// Returns true if the next element looks like an encoded dict
bool is_dict() { return consume_key() && data.front() == 'd'; }
/// Returns the key of the next pair. This does not have to be called; it is also returned by
/// all of the other consume_* methods. The value is cached whether called here or by some
/// other method; accessing it multiple times simple accesses the cache until the next value is
/// consumed.
std::string_view key() {
if (!consume_key())
throw bt_deserialize_invalid{"Cannot access next key: at the end of the dict"};
return key_;
}
/// Attempt to parse the next value as a string->string pair (and advance just past it). Throws
/// if the next value is not a string.
std::pair<std::string_view, std::string_view> next_string();
/// Attempts to parse the next value as an string->integer pair (and advance just past it).
/// Throws if the next value is not an integer.
template <typename IntType>
std::pair<std::string_view, IntType> next_integer() {
if (!is_integer()) throw bt_deserialize_invalid_type{"next bt dict value is not an integer"};
std::pair<std::string_view, IntType> ret;
ret.second = bt_list_consumer::consume_integer<IntType>();
ret.first = flush_key();
return ret;
}
/// Consumes a string->list pair, return it as a list-like type. This typically requires
/// dynamic allocation, but only has to parse the data once. Compare with consume_list_data()
/// which allows alloc-free traversal, but requires parsing twice (if the contents are to be
/// used).
template <typename T = bt_list>
std::pair<std::string_view, T> next_list() {
std::pair<std::string_view, T> pair;
pair.first = next_list(pair.second);
return pair;
}
/// Same as above, but takes a pre-existing list-like data type. Returns the key.
template <typename T>
std::string_view next_list(T& list) {
if (!is_list()) throw bt_deserialize_invalid_type{"next bt value is not a list"};
bt_list_consumer::consume_list(list);
return flush_key();
}
/// Consumes a string->dict pair, return it as a dict-like type. This typically requires
/// dynamic allocation, but only has to parse the data once. Compare with consume_dict_data()
/// which allows alloc-free traversal, but requires parsing twice (if the contents are to be
/// used).
template <typename T = bt_dict>
std::pair<std::string_view, T> next_dict() {
std::pair<std::string_view, T> pair;
pair.first = consume_dict(pair.second);
return pair;
}
/// Same as above, but takes a pre-existing dict-like data type. Returns the key.
template <typename T>
std::string_view next_dict(T& dict) {
if (!is_dict()) throw bt_deserialize_invalid_type{"next bt value is not a dict"};
bt_list_consumer::consume_dict(dict);
return flush_key();
}
/// Attempts to parse the next value as a string->list pair and returns the string_view that
/// contains the entire thing. This is recursive into both lists and dicts and likely to be
/// quite inefficient for large, nested structures (unless the values only need to be skipped
/// but aren't separately needed). This, however, does not require dynamic memory allocation.
std::pair<std::string_view, std::string_view> next_list_data() {
if (data.size() < 2 || !is_list()) throw bt_deserialize_invalid_type{"next bt dict value is not a list"};
return {flush_key(), bt_list_consumer::consume_list_data()};
}
/// Same as next_list_data(), but wraps the value in a bt_list_consumer for convenience
std::pair<std::string_view, bt_list_consumer> next_list_consumer() { return next_list_data(); }
/// Attempts to parse the next value as a string->dict pair and returns the string_view that
/// contains the entire thing. This is recursive into both lists and dicts and likely to be
/// quite inefficient for large, nested structures (unless the values only need to be skipped
/// but aren't separately needed). This, however, does not require dynamic memory allocation.
std::pair<std::string_view, std::string_view> next_dict_data() {
if (data.size() < 2 || !is_dict()) throw bt_deserialize_invalid_type{"next bt dict value is not a dict"};
return {flush_key(), bt_list_consumer::consume_dict_data()};
}
/// Same as next_dict_data(), but wraps the value in a bt_dict_consumer for convenience
std::pair<std::string_view, bt_dict_consumer> next_dict_consumer() { return next_dict_data(); }
/// Skips ahead until we find the first key >= the given key or reach the end of the dict.
/// Returns true if we found an exact match, false if we reached some greater value or the end.
/// If we didn't hit the end, the next `consumer_*()` call will return the key-value pair we
/// found (either the exact match or the first key greater than the requested key).
///
/// Two important notes:
///
/// - properly encoded bt dicts must have lexicographically sorted keys, and this method assumes
/// that the input is correctly sorted (and thus if we find a greater value then your key does
/// not exist).
/// - this is irreversible; you cannot returned to skipped values without reparsing. (You *can*
/// however, make a copy of the bt_dict_consumer before calling and use the copy to return to
/// the pre-skipped position).
bool skip_until(std::string_view find) {
while (consume_key() && key_ < find) {
flush_key();
skip_value();
}
return key_ == find;
}
/// The `consume_*` functions are wrappers around next_whatever that discard the returned key.
///
/// Intended for use with skip_until such as:
///
/// std::string value;
/// if (d.skip_until("key"))
/// value = d.consume_string();
///
auto consume_string_view() { return next_string().second; }
auto consume_string() { return std::string{consume_string_view()}; }
template <typename IntType>
auto consume_integer() { return next_integer<IntType>().second; }
template <typename T = bt_list>
auto consume_list() { return next_list<T>().second; }
template <typename T>
void consume_list(T& list) { next_list(list); }
template <typename T = bt_dict>
auto consume_dict() { return next_dict<T>().second; }
template <typename T>
void consume_dict(T& dict) { next_dict(dict); }
std::string_view consume_list_data() { return next_list_data().second; }
std::string_view consume_dict_data() { return next_dict_data().second; }
bt_list_consumer consume_list_consumer() { return consume_list_data(); }
bt_dict_consumer consume_dict_consumer() { return consume_dict_data(); }
};
} // namespace lokimq
namespace lokimq = oxenmq;

112
lokimq/bt_value.h
View File

@ -1,112 +1,4 @@
// Copyright (c) 2019-2020, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include "../oxenmq/bt_value.h"
// This header is here to provide just the basic bt_value/bt_dict/bt_list definitions without
// needing to include the full bt_serialize.h header.
#include <map>
#include <list>
#include <cstdint>
#include <variant>
#include <string>
#include <string_view>
namespace lokimq {
struct bt_value;
/// The type used to store dictionaries inside bt_value.
using bt_dict = std::map<std::string, bt_value>; // NB: unordered_map doesn't work because it can't be used with a predeclared type
/// The type used to store list items inside bt_value.
using bt_list = std::list<bt_value>;
/// The basic variant that can hold anything (recursively).
using bt_variant = std::variant<
std::string,
std::string_view,
int64_t,
uint64_t,
bt_list,
bt_dict
>;
#ifdef __cpp_lib_remove_cvref // C++20
using std::remove_cvref_t;
#else
template <typename T>
using remove_cvref_t = std::remove_cv_t<std::remove_reference_t<T>>;
#endif
template <typename T, typename Variant>
struct has_alternative;
template <typename T, typename... V>
struct has_alternative<T, std::variant<V...>> : std::bool_constant<(std::is_same_v<T, V> || ...)> {};
template <typename T, typename Variant>
constexpr bool has_alternative_v = has_alternative<T, Variant>::value;
namespace detail {
template <typename Tuple, size_t... Is>
bt_list tuple_to_list(const Tuple& tuple, std::index_sequence<Is...>) {
return {{bt_value{std::get<Is>(tuple)}...}};
}
template <typename T> constexpr bool is_tuple = false;
template <typename... T> constexpr bool is_tuple<std::tuple<T...>> = true;
template <typename S, typename T> constexpr bool is_tuple<std::pair<S, T>> = true;
}
/// Recursive generic type that can fully represent everything valid for a BT serialization.
/// This is basically just an empty wrapper around the std::variant, except we add some extra
/// converting constructors:
/// - integer constructors so that any unsigned value goes to the uint64_t and any signed value goes
/// to the int64_t.
/// - std::tuple and std::pair constructors that build a bt_list out of the tuple/pair elements.
struct bt_value : bt_variant {
using bt_variant::bt_variant;
using bt_variant::operator=;
template <typename T, typename U = std::remove_reference_t<T>, std::enable_if_t<std::is_integral_v<U> && std::is_unsigned_v<U>, int> = 0>
bt_value(T&& uint) : bt_variant{static_cast<uint64_t>(uint)} {}
template <typename T, typename U = std::remove_reference_t<T>, std::enable_if_t<std::is_integral_v<U> && std::is_signed_v<U>, int> = 0>
bt_value(T&& sint) : bt_variant{static_cast<int64_t>(sint)} {}
template <typename... T>
bt_value(const std::tuple<T...>& tuple) : bt_variant{detail::tuple_to_list(tuple, std::index_sequence_for<T...>{})} {}
template <typename S, typename T>
bt_value(const std::pair<S, T>& pair) : bt_variant{detail::tuple_to_list(pair, std::index_sequence_for<S, T>{})} {}
template <typename T, typename U = std::remove_reference_t<T>, std::enable_if_t<!std::is_integral_v<U> && !detail::is_tuple<U>, int> = 0>
bt_value(T&& v) : bt_variant{std::forward<T>(v)} {}
bt_value(const char* s) : bt_value{std::string_view{s}} {}
};
}
namespace lokimq = oxenmq;

96
lokimq/connections.h
View File

@ -1,96 +1,4 @@
#pragma once
#include "auth.h"
#include "bt_value.h"
#include <string_view>
#include <iosfwd>
#include <stdexcept>
#include <string>
#include <utility>
#include <variant>
namespace lokimq {
struct ConnectionID;
namespace detail {
template <typename... T>
bt_dict build_send(ConnectionID to, std::string_view cmd, T&&... opts);
}
/// Opaque data structure representing a connection which supports ==, !=, < and std::hash. For
/// connections to service node this is the service node pubkey (and you can pass a 32-byte string
/// anywhere a ConnectionID is called for). For non-SN remote connections you need to keep a copy
/// of the ConnectionID returned by connect_remote().
struct ConnectionID {
// Default construction; creates a ConnectionID with an invalid internal ID that will not match
// an actual connection.
ConnectionID() : ConnectionID(0) {}
// Construction from a service node pubkey
ConnectionID(std::string pubkey_) : id{SN_ID}, pk{std::move(pubkey_)} {
if (pk.size() != 32)
throw std::runtime_error{"Invalid pubkey: expected 32 bytes"};
}
// Construction from a service node pubkey
ConnectionID(std::string_view pubkey_) : ConnectionID(std::string{pubkey_}) {}
ConnectionID(const ConnectionID&) = default;
ConnectionID(ConnectionID&&) = default;
ConnectionID& operator=(const ConnectionID&) = default;
ConnectionID& operator=(ConnectionID&&) = default;
// Returns true if this is a ConnectionID (false for a default-constructed, invalid id)
explicit operator bool() const {
return id != 0;
}
// Two ConnectionIDs are equal if they are both SNs and have matching pubkeys, or they are both
// not SNs and have matching internal IDs and routes. (Pubkeys do not have to match for
// non-SNs).
bool operator==(const ConnectionID &o) const {
if (sn() && o.sn())
return pk == o.pk;
return id == o.id && route == o.route;
}
bool operator!=(const ConnectionID &o) const { return !(*this == o); }
bool operator<(const ConnectionID &o) const {
if (sn() && o.sn())
return pk < o.pk;
return id < o.id || (id == o.id && route < o.route);
}
// Returns true if this ConnectionID represents a SN connection
bool sn() const { return id == SN_ID; }
// Returns this connection's pubkey, if any. (Note that all curve connections have pubkeys, not
// only SNs).
const std::string& pubkey() const { return pk; }
// Returns a copy of the ConnectionID with the route set to empty.
ConnectionID unrouted() { return ConnectionID{id, pk, ""}; }
private:
ConnectionID(long long id) : id{id} {}
ConnectionID(long long id, std::string pubkey, std::string route = "")
: id{id}, pk{std::move(pubkey)}, route{std::move(route)} {}
constexpr static long long SN_ID = -1;
long long id = 0;
std::string pk;
std::string route;
friend class LokiMQ;
friend struct std::hash<ConnectionID>;
template <typename... T>
friend bt_dict detail::build_send(ConnectionID to, std::string_view cmd, T&&... opts);
friend std::ostream& operator<<(std::ostream& o, const ConnectionID& conn);
};
} // namespace lokimq
namespace std {
template <> struct hash<lokimq::ConnectionID> {
size_t operator()(const lokimq::ConnectionID &c) const {
return c.sn() ? lokimq::already_hashed{}(c.pk) :
std::hash<long long>{}(c.id) + std::hash<std::string>{}(c.route);
}
};
} // namespace std
#include "../oxenmq/connections.h"
namespace lokimq = oxenmq;

145
lokimq/hex.h
View File

@ -1,145 +1,4 @@
// Copyright (c) 2019-2020, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <string>
#include <string_view>
#include <array>
#include <iterator>
#include <cassert>
#include "../oxenmq/hex.h"
namespace lokimq {
namespace detail {
/// Compile-time generated lookup tables hex conversion
struct hex_table {
char from_hex_lut[256];
char to_hex_lut[16];
constexpr hex_table() noexcept : from_hex_lut{}, to_hex_lut{} {
for (unsigned char c = 0; c < 10; c++) {
from_hex_lut[(unsigned char)('0' + c)] = 0 + c;
to_hex_lut[ (unsigned char)( 0 + c)] = '0' + c;
}
for (unsigned char c = 0; c < 6; c++) {
from_hex_lut[(unsigned char)('a' + c)] = 10 + c;
from_hex_lut[(unsigned char)('A' + c)] = 10 + c;
to_hex_lut[ (unsigned char)(10 + c)] = 'a' + c;
}
}
constexpr char from_hex(unsigned char c) const noexcept { return from_hex_lut[c]; }
constexpr char to_hex(unsigned char b) const noexcept { return to_hex_lut[b]; }
} constexpr hex_lut;
// This main point of this static assert is to force the compiler to compile-time build the constexpr tables.
static_assert(hex_lut.from_hex('a') == 10 && hex_lut.from_hex('F') == 15 && hex_lut.to_hex(13) == 'd', "");
} // namespace detail
/// Creates hex digits from a character sequence.
template <typename InputIt, typename OutputIt>
void to_hex(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "to_hex requires chars/bytes");
for (; begin != end; ++begin) {
uint8_t c = static_cast<uint8_t>(*begin);
*out++ = detail::hex_lut.to_hex(c >> 4);
*out++ = detail::hex_lut.to_hex(c & 0x0f);
}
}
/// Creates a string of hex digits from a character sequence iterator pair
template <typename It>
std::string to_hex(It begin, It end) {
std::string hex;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>)
hex.reserve(2 * std::distance(begin, end));
to_hex(begin, end, std::back_inserter(hex));
return hex;
}
/// Creates a hex string from an iterable, std::string-like object
template <typename CharT>
std::string to_hex(std::basic_string_view<CharT> s) { return to_hex(s.begin(), s.end()); }
inline std::string to_hex(std::string_view s) { return to_hex<>(s); }
/// Returns true if all elements in the range are hex characters
template <typename It>
constexpr bool is_hex(It begin, It end) {
static_assert(sizeof(decltype(*begin)) == 1, "is_hex requires chars/bytes");
for (; begin != end; ++begin) {
if (detail::hex_lut.from_hex(static_cast<unsigned char>(*begin)) == 0 && static_cast<unsigned char>(*begin) != '0')
return false;
}
return true;
}
/// Returns true if all elements in the string-like value are hex characters
template <typename CharT>
constexpr bool is_hex(std::basic_string_view<CharT> s) { return is_hex(s.begin(), s.end()); }
constexpr bool is_hex(std::string_view s) { return is_hex(s.begin(), s.end()); }
/// Convert a hex digit into its numeric (0-15) value
constexpr char from_hex_digit(unsigned char x) noexcept {
return detail::hex_lut.from_hex(x);
}
/// Constructs a byte value from a pair of hex digits
constexpr char from_hex_pair(unsigned char a, unsigned char b) noexcept { return (from_hex_digit(a) << 4) | from_hex_digit(b); }
/// Converts a sequence of hex digits to bytes. Undefined behaviour if any characters are not in
/// [0-9a-fA-F] or if the input sequence length is not even. It is permitted for the input and
/// output ranges to overlap as long as out is no earlier than begin.
template <typename InputIt, typename OutputIt>
void from_hex(InputIt begin, InputIt end, OutputIt out) {
using std::distance;
assert(distance(begin, end) % 2 == 0);
while (begin != end) {
auto a = *begin++;
auto b = *begin++;
*out++ = from_hex_pair(static_cast<unsigned char>(a), static_cast<unsigned char>(b));
}
}
/// Converts a sequence of hex digits to a string of bytes and returns it. Undefined behaviour if
/// the input sequence is not an even-length sequence of [0-9a-fA-F] characters.
template <typename It>
std::string from_hex(It begin, It end) {
std::string bytes;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>)
bytes.reserve(std::distance(begin, end) / 2);
from_hex(begin, end, std::back_inserter(bytes));
return bytes;
}
/// Converts hex digits from a std::string-like object into a std::string of bytes. Undefined
/// behaviour if any characters are not in [0-9a-fA-F] or if the input sequence length is not even.
template <typename CharT>
std::string from_hex(std::basic_string_view<CharT> s) { return from_hex(s.begin(), s.end()); }
inline std::string from_hex(std::string_view s) { return from_hex<>(s); }
}
namespace lokimq = oxenmq;

1526
lokimq/lokimq.h
View File

File diff suppressed because it is too large Load Diff

57
lokimq/message.h
View File

@ -1,57 +1,4 @@
#pragma once
#include <vector>
#include "connections.h"
#include "../oxenmq/message.h"
namespace lokimq {
class LokiMQ;
/// Encapsulates an incoming message from a remote connection with message details plus extra
/// info need to send a reply back through the proxy thread via the `reply()` method. Note that
/// this object gets reused: callbacks should use but not store any reference beyond the callback.
class Message {
public:
LokiMQ& lokimq; ///< The owning LokiMQ object
std::vector<std::string_view> data; ///< The provided command data parts, if any.
ConnectionID conn; ///< The connection info for routing a reply; also contains the pubkey/sn status.
std::string reply_tag; ///< If the invoked command is a request command this is the required reply tag that will be prepended by `send_reply()`.
Access access; ///< The access level of the invoker. This can be higher than the access level of the command, for example for an admin invoking a basic command.
std::string remote; ///< Some sort of remote address from which the request came. Often "IP" for TCP connections and "localhost:UID:GID:PID" for UDP connections.
/// Constructor
Message(LokiMQ& lmq, ConnectionID cid, Access access, std::string remote)
: lokimq{lmq}, conn{std::move(cid)}, access{std::move(access)}, remote{std::move(remote)} {}
// Non-copyable
Message(const Message&) = delete;
Message& operator=(const Message&) = delete;
/// Sends a command back to whomever sent this message. Arguments are forwarded to send() but
/// with send_option::optional{} added if the originator is not a SN. For SN messages (i.e.
/// where `sn` is true) this is a "strong" reply by default in that the proxy will attempt to
/// establish a new connection to the SN if no longer connected. For non-SN messages the reply
/// will be attempted using the available routing information, but if the connection has already
/// been closed the reply will be dropped.
///
/// If you want to send a non-strong reply even when the remote is a service node then add
/// an explicit `send_option::optional()` argument.
template <typename... Args>
void send_back(std::string_view, Args&&... args);
/// Sends a reply to a request. This takes no command: the command is always the built-in
/// "REPLY" command, followed by the unique reply tag, then any reply data parts. All other
/// arguments are as in `send_back()`. You should only send one reply for a command expecting
/// replies, though this is not enforced: attempting to send multiple replies will simply be
/// dropped when received by the remote. (Note, however, that it is possible to send multiple
/// messages -- e.g. you could send a reply and then also call send_back() and/or send_request()
/// to send more requests back to the sender).
template <typename... Args>
void send_reply(Args&&... args);
/// Sends a request back to whomever sent this message. This is effectively a wrapper around
/// lmq.request() that takes care of setting up the recipient arguments.
template <typename ReplyCallback, typename... Args>
void send_request(std::string_view cmd, ReplyCallback&& callback, Args&&... args);
};
}
namespace lokimq = oxenmq;

15
lokimq/string_view.h
View File

@ -1,15 +0,0 @@
#pragma once
#include <string_view>
namespace lokimq {
// Deprecated type alias for std::string_view
using string_view = std::string_view;
// Deprecated "foo"_sv literal; you should use "foo"sv (from <string_view>) instead.
inline namespace literals {
inline constexpr std::string_view operator""_sv(const char* str, size_t len) { return {str, len}; }
}
}

103
lokimq/variant.h
View File

@ -1,103 +1,2 @@
#pragma once
// Workarounds for macos compatibility. On macOS we aren't allowed to touch anything in
// std::variant that could throw if compiling with a target <10.14 because Apple fails hard at
// properly updating their STL. Thus, if compiling in such a mode, we have to introduce
// workarounds.
//
// This header defines a `var` namespace with `var::get` and `var::visit` implementations. On
// everything except broken backwards macos, this is just an alias to `std`. On broken backwards
// macos, we provide implementations that throw std::runtime_error in failure cases since the
// std::bad_variant_access exception can't be touched.
//
// You also get a BROKEN_APPLE_VARIANT macro defined if targetting a problematic mac architecture.
#include <variant>
#ifdef __APPLE__
# include <AvailabilityVersions.h>
# if defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_14
# define BROKEN_APPLE_VARIANT
# endif
#endif
#ifndef BROKEN_APPLE_VARIANT
namespace var = std; // Oh look, actual C++17 support
#else
// Oh look, apple.
namespace var {
// Apple won't let us use std::visit or std::get if targetting some version of macos earlier than
// 10.14 because Apple is awful about not updating their STL. So we have to provide our own, and
// then call these without `std::` -- on crappy macos we'll come here, on everything else we'll ADL
// to the std:: implementation.
template <typename T, typename... Types>
constexpr T& get(std::variant<Types...>& var) {
if (auto* v = std::get_if<T>(&var)) return *v;
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <typename T, typename... Types>
constexpr const T& get(const std::variant<Types...>& var) {
if (auto* v = std::get_if<T>(&var)) return *v;
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <typename T, typename... Types>
constexpr const T&& get(const std::variant<Types...>&& var) {
if (auto* v = std::get_if<T>(&var)) return std::move(*v);
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <typename T, typename... Types>
constexpr T&& get(std::variant<Types...>&& var) {
if (auto* v = std::get_if<T>(&var)) return std::move(*v);
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, typename... Types>
constexpr auto& get(std::variant<Types...>& var) {
if (auto* v = std::get_if<I>(&var)) return *v;
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, typename... Types>
constexpr const auto& get(const std::variant<Types...>& var) {
if (auto* v = std::get_if<I>(&var)) return *v;
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, typename... Types>
constexpr const auto&& get(const std::variant<Types...>&& var) {
if (auto* v = std::get_if<I>(&var)) return std::move(*v);
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, typename... Types>
constexpr auto&& get(std::variant<Types...>&& var) {
if (auto* v = std::get_if<I>(&var)) return std::move(*v);
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, size_t... More, class Visitor, class Variant>
constexpr auto visit_helper(Visitor&& vis, Variant&& var) {
if (var.index() == I)
return vis(var::get<I>(std::forward<Variant>(var)));
else if constexpr (sizeof...(More) > 0)
return visit_helper<More...>(std::forward<Visitor>(vis), std::forward<Variant>(var));
else
throw std::runtime_error{"Bad visit -- variant is valueless"};
}
template <size_t... Is, class Visitor, class Variant>
constexpr auto visit_helper(Visitor&& vis, Variant&& var, std::index_sequence<Is...>) {
return visit_helper<Is...>(std::forward<Visitor>(vis), std::forward<Variant>(var));
}
// Only handle a single variant here because multi-variant invocation is notably harder (and we
// don't need it).
template <class Visitor, class Variant>
constexpr auto visit(Visitor&& vis, Variant&& var) {
return visit_helper(std::forward<Visitor>(vis), std::forward<Variant>(var),
std::make_index_sequence<std::variant_size_v<std::remove_reference_t<Variant>>>{});
}
} // namespace var
#endif
#include "../oxenmq/variant.h"

4
lokimq/version.h Normal file
View File

@ -0,0 +1,4 @@
#pragma once
#include "../oxenmq/version.h"
namespace lokimq = oxenmq;

5
lokimq/version.h.in
View File

@ -1,5 +0,0 @@
namespace lokimq {
constexpr int VERSION_MAJOR = @LOKIMQ_VERSION_MAJOR@;
constexpr int VERSION_MINOR = @LOKIMQ_VERSION_MINOR@;
constexpr int VERSION_PATCH = @LOKIMQ_VERSION_PATCH@;
}

8
lokimq/address.cpp → oxenmq/address.cpp
View File

@ -9,7 +9,7 @@
#include "base32z.h"
#include "base64.h"
namespace lokimq {
namespace oxenmq {
constexpr size_t enc_length(address::encoding enc) {
return enc == address::encoding::hex ? 64 :
@ -23,13 +23,13 @@ constexpr size_t enc_length(address::encoding enc) {
// given: for QR-friendly we only accept hex or base32z (since QR cannot handle base64's alphabet).
std::string decode_pubkey(std::string_view& in, bool qr) {
std::string pubkey;
if (in.size() >= 64 && lokimq::is_hex(in.substr(0, 64))) {
if (in.size() >= 64 && is_hex(in.substr(0, 64))) {
pubkey = from_hex(in.substr(0, 64));
in.remove_prefix(64);
} else if (in.size() >= 52 && lokimq::is_base32z(in.substr(0, 52))) {
} else if (in.size() >= 52 && is_base32z(in.substr(0, 52))) {
pubkey = from_base32z(in.substr(0, 52));
in.remove_prefix(52);
} else if (!qr && in.size() >= 43 && lokimq::is_base64(in.substr(0, 43))) {
} else if (!qr && in.size() >= 43 && is_base64(in.substr(0, 43))) {
pubkey = from_base64(in.substr(0, 43));
in.remove_prefix(43);
if (!in.empty() && in.front() == '=')

210
oxenmq/address.h Normal file
View File

@ -0,0 +1,210 @@
// Copyright (c) 2020-2021, The Oxen Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <string>
#include <string_view>
#include <cstdint>
#include <iosfwd>
namespace oxenmq {
using namespace std::literals;
/** OxenMQ address abstraction class. This class uses and extends standard ZMQ addresses allowing
* extra parameters to be passed in in a relative standard way.
*
* External ZMQ addresses generally have two forms that we are concerned with: one for TCP and one
* for Unix sockets:
*
* tcp://HOST:PORT -- HOST can be a hostname, IPv4 address, or IPv6 address in [...]
* ipc://PATH -- PATH can be absolute (ipc:///path/to/some.sock) or relative (ipc://some.sock)
*
* but this doesn't carry enough info: in particular, we can connect with two very different
* protocols: curve25519-encrypted, or plaintext, but for curve25519-encrypted we require the
* remote's public key as well to verify the connection.
*
* This class, then, handles this by allowing addresses of:
*
* Standard ZMQ address: these carry no pubkey and so the connection will be unencrypted:
*
* tcp://HOSTNAME:PORT
* ipc://PATH
*
* Non-ZMQ address formats that specify that the connection shall be x25519 encrypted:
*
* curve://HOSTNAME:PORT/PUBKEY -- PUBKEY must be specified in hex (64 characters), base32z (52)
* or base64 (43 or 44 with one '=' trailing padding)
* ipc+curve:///path/to/my.sock/PUBKEY -- same requirements on PUBKEY as above.
* tcp+curve://(whatever) -- alias for curve://(whatever)
*
* We also accept special upper-case TCP-only variants which *only* accept uppercase characters and
* a few required symbols (:, /, $, ., and -) in the string:
*
* TCP://HOSTNAME:PORT
* CURVE://HOSTNAME:PORT/B32ZPUBKEY
*
* These versions are explicitly meant to be used with QR codes; the upper-case-only requirement
* allows a smaller QR code by allowing QR's alphanumeric mode (which allows only [A-Z0-9 $%*+./:-])
* to be used. Such a QR-friendly address can be created from the qr_address() method. To support
* literal IPv6 addresses we surround the address with $...$ instead of the usual [...].
*
* Note that this class does very little validate the host argument at all, and no socket path
* validation whatsoever. The only constraint on host is when parsing an encoded address: we check
* that it contains no : at all, or must be a [bracketed] expression that contains only hex
* characters, :'s, or .'s. Otherwise, if you pass broken crap into the hostname, expect broken
* crap out.
*/
struct address {
/// Supported address protocols: TCP connections (tcp), or unix sockets (ipc).
enum class proto {
tcp,
tcp_curve,
ipc,
ipc_curve
};
/// Supported public key encodings (used when regenerating an augmented address).
enum class encoding {
hex, ///< hexadecimal encoded
base32z, ///< base32z encoded
base64, ///< base64 encoded (*without* trailing = padding)
BASE32Z ///< upper-case base32z encoding, meant for QR encoding
};
/// The protocol: one of the `protocol` enum values for tcp or ipc (unix sockets), with or
/// without _curve encryption.
proto protocol = proto::tcp;
/// The host for tcp connections; can be a hostname or IP address. If this is an IPv6 it must be surrounded with [ ].
std::string host;
/// The port (for tcp connections)
uint16_t port = 0;
/// The socket path (for unix socket connections)
std::string socket;
/// If a curve connection, this is the required remote public key (in bytes)
std::string pubkey;
/// Default constructor; this gives you an unusable address.
address() = default;
/**
* Constructs an address by parsing a string_view containing one of the formats listed in the
* class description. This is intentionally implicitly constructible so that you can pass a
* string_view into anything expecting an `address`.
*
* Throw std::invalid_argument if the given address is not parseable.
*/
address(std::string_view addr);
/** Constructs an address from a remote string and a separate pubkey. Typically `remote` is a
* basic ZMQ connect string, though this is not enforced. Any pubkey information embedded in
* the remote string will be discarded and replaced with the given pubkey string. The result
* will be curve encrypted if `pubkey` is non-empty, plaintext if `pubkey` is empty.
*
* Throws an exception if either addr or pubkey is invalid.
*
* Exactly equivalent to `address a{remote}; a.set_pubkey(pubkey);`
*/
address(std::string_view addr, std::string_view pubkey) : address(addr) { set_pubkey(pubkey); }
/// Replaces the address's pubkey (if any) with the given pubkey (or no pubkey if empty). If
/// changing from pubkey to no-pubkey or no-pubkey to pubkey then the protocol is update to
/// switch to or from curve encryption.
///
/// pubkey should be the 32-byte binary pubkey, or an empty string to remove an existing pubkey.
///
/// Returns the object itself, so that you can chain it.
address& set_pubkey(std::string_view pubkey);
/// Constructs and builds the ZMQ connection address from the stored connection details. This
/// does not contain any of the curve-related details; those must be specified separately when
/// interfacing with ZMQ.
std::string zmq_address() const;
/// Returns true if the connection was specified as a curve-encryption-enabled connection, false
/// otherwise.
bool curve() const { return protocol == proto::tcp_curve || protocol == proto::ipc_curve; }
/// True if the protocol is TCP (either with or without curve)
bool tcp() const { return protocol == proto::tcp || protocol == proto::tcp_curve; }
/// True if the protocol is unix socket (either with or without curve)
bool ipc() const { return !tcp(); }
/// Returns the full "augmented" address string (i.e. that could be passed in to the
/// constructor). This will be equivalent (but not necessarily identical) to an augmented
/// string passed into the constructor. Takes an optional encoding format for the pubkey (if
/// any), which defaults to base32z.
std::string full_address(encoding enc = encoding::base32z) const;
/// Returns a QR-code friendly address string. This returns an all-uppercase version of the
/// address with "TCP://" or "CURVE://" for the protocol string, and uses upper-case base32z
/// encoding for the pubkey (for curve addresses). For literal IPv6 addresses we replace the
/// surround the
/// address with $ instead of $
///
/// \throws std::logic_error if called on a unix socket address.
std::string qr_address() const;
/// Returns `.pubkey` but encoded in the given format
std::string encode_pubkey(encoding enc) const;
/// Returns true if two addresses are identical (i.e. same protocol and relevant protocol
/// arguments).
///
/// Note that it is possible for addresses to connect to the same socket without being
/// identical: for example, using "foo.sock" and "./foo.sock", or writing IPv6 addresses (or
/// even IPv4 addresses) in slightly different ways). Such equivalent but non-equal values will
/// result in a false return here.
///
/// Note also that we ignore irrelevant arguments: for example, we don't care whether pubkeys
/// match when comparing two non-curve TCP addresses.
bool operator==(const address& other) const;
/// Negation of ==
bool operator!=(const address& other) const { return !operator==(other); }
/// Factory function that constructs a TCP address from a host and port. The connection will be
/// plaintext. If the host is an IPv6 address it *must* be surrounded with [ and ].
static address tcp(std::string host, uint16_t port);
/// Factory function that constructs a curve-encrypted TCP address from a host, port, and remote
/// pubkey. The pubkey must be 32 bytes. As above, IPv6 addresses must be specified as [addr].
static address tcp_curve(std::string host, uint16_t, std::string pubkey);
/// Factory function that constructs a unix socket address from a path. The connection will be
/// plaintext (which is usually fine for a socket since unix sockets are local machine).
static address ipc(std::string path);
/// Factory function that constructs a unix socket address from a path and remote pubkey. The
/// connection will be curve25519 encrypted; the remote pubkey must be 32 bytes.
static address ipc_curve(std::string path, std::string pubkey);
};
// Outputs address.full_address() when sent to an ostream.
std::ostream& operator<<(std::ostream& o, const address& a);
}

26
lokimq/auth.cpp → oxenmq/auth.cpp
View File

@ -1,10 +1,10 @@
#include "lokimq.h"
#include "oxenmq.h"
#include "hex.h"
#include "lokimq-internal.h"
#include "oxenmq-internal.h"
#include <ostream>
#include <sstream>
namespace lokimq {
namespace oxenmq {
std::ostream& operator<<(std::ostream& o, AuthLevel a) {
return o << to_string(a);
@ -31,7 +31,7 @@ std::string zmtp_metadata(std::string_view key, std::string_view value) {
}
bool LokiMQ::proxy_check_auth(size_t conn_index, bool outgoing, const peer_info& peer,
bool OxenMQ::proxy_check_auth(size_t conn_index, bool outgoing, const peer_info& peer,
zmq::message_t& cmd, const cat_call_t& cat_call, std::vector<zmq::message_t>& data) {
auto command = view(cmd);
std::string reply;
@ -45,7 +45,7 @@ bool LokiMQ::proxy_check_auth(size_t conn_index, bool outgoing, const peer_info&
reply = "FORBIDDEN";
} else if (cat_call.first->access.local_sn && !local_service_node) {
LMQ_LOG(warn, "Access denied to ", command, " for peer [", to_hex(peer.pubkey), "]/", peer_address(cmd),
": that command is only available when this LokiMQ is running in service node mode");
": that command is only available when this OxenMQ is running in service node mode");
reply = "NOT_A_SERVICE_NODE";
} else if (cat_call.first->access.remote_sn && !peer.service_node) {
LMQ_LOG(warn, "Access denied to ", command, " for peer [", to_hex(peer.pubkey), "]/", peer_address(cmd),
@ -81,7 +81,7 @@ bool LokiMQ::proxy_check_auth(size_t conn_index, bool outgoing, const peer_info&
return false;
}
void LokiMQ::set_active_sns(pubkey_set pubkeys) {
void OxenMQ::set_active_sns(pubkey_set pubkeys) {
if (proxy_thread.joinable()) {
auto data = bt_serialize(detail::serialize_object(std::move(pubkeys)));
detail::send_control(get_control_socket(), "SET_SNS", data);
@ -89,10 +89,10 @@ void LokiMQ::set_active_sns(pubkey_set pubkeys) {
proxy_set_active_sns(std::move(pubkeys));
}
}
void LokiMQ::proxy_set_active_sns(std::string_view data) {
void OxenMQ::proxy_set_active_sns(std::string_view data) {
proxy_set_active_sns(detail::deserialize_object<pubkey_set>(bt_deserialize<uintptr_t>(data)));
}
void LokiMQ::proxy_set_active_sns(pubkey_set pubkeys) {
void OxenMQ::proxy_set_active_sns(pubkey_set pubkeys) {
pubkey_set added, removed;
for (auto it = pubkeys.begin(); it != pubkeys.end(); ) {
auto& pk = *it;
@ -118,7 +118,7 @@ void LokiMQ::proxy_set_active_sns(pubkey_set pubkeys) {
proxy_update_active_sns_clean(std::move(added), std::move(removed));
}
void LokiMQ::update_active_sns(pubkey_set added, pubkey_set removed) {
void OxenMQ::update_active_sns(pubkey_set added, pubkey_set removed) {
LMQ_LOG(info, "uh, ", added.size());
if (proxy_thread.joinable()) {
std::array<uintptr_t, 2> data;
@ -129,12 +129,12 @@ void LokiMQ::update_active_sns(pubkey_set added, pubkey_set removed) {
proxy_update_active_sns(std::move(added), std::move(removed));
}
}
void LokiMQ::proxy_update_active_sns(bt_list_consumer data) {
void OxenMQ::proxy_update_active_sns(bt_list_consumer data) {
auto added = detail::deserialize_object<pubkey_set>(data.consume_integer<uintptr_t>());
auto remed = detail::deserialize_object<pubkey_set>(data.consume_integer<uintptr_t>());
proxy_update_active_sns(std::move(added), std::move(remed));
}
void LokiMQ::proxy_update_active_sns(pubkey_set added, pubkey_set removed) {
void OxenMQ::proxy_update_active_sns(pubkey_set added, pubkey_set removed) {
// We take a caller-provided set of added/removed then filter out any junk (bad pks, conflicting
// values, pubkeys that already(added) or do not(removed) exist), then pass the purified lists
// to the _clean version.
@ -167,7 +167,7 @@ void LokiMQ::proxy_update_active_sns(pubkey_set added, pubkey_set removed) {
proxy_update_active_sns_clean(std::move(added), std::move(removed));
}
void LokiMQ::proxy_update_active_sns_clean(pubkey_set added, pubkey_set removed) {
void OxenMQ::proxy_update_active_sns_clean(pubkey_set added, pubkey_set removed) {
LMQ_LOG(debug, "Updating SN auth status with +", added.size(), "/-", removed.size(), " pubkeys");
// For anything we remove we want close the connection to the SN (if outgoing), and remove the
@ -192,7 +192,7 @@ void LokiMQ::proxy_update_active_sns_clean(pubkey_set added, pubkey_set removed)
active_service_nodes.insert(std::move(pk));
}
void LokiMQ::process_zap_requests() {
void OxenMQ::process_zap_requests() {
for (std::vector<zmq::message_t> frames; recv_message_parts(zap_auth, frames, zmq::recv_flags::dontwait); frames.clear()) {
#ifndef NDEBUG
if (log_level() >= LogLevel::trace) {

55
oxenmq/auth.h Normal file
View File

@ -0,0 +1,55 @@
#pragma once
#include <iosfwd>
#include <string>
#include <cstring>
#include <unordered_set>
namespace oxenmq {
/// Authentication levels for command categories and connections
enum class AuthLevel {
denied, ///< Not actually an auth level, but can be returned by the AllowFunc to deny an incoming connection.
none, ///< No authentication at all; any random incoming ZMQ connection can invoke this command.
basic, ///< Basic authentication commands require a login, or a node that is specifically configured to be a public node (e.g. for public RPC).
admin, ///< Advanced authentication commands require an admin user, either via explicit login or by implicit login from localhost. This typically protects administrative commands like shutting down, starting mining, or access sensitive data.
};
std::ostream& operator<<(std::ostream& os, AuthLevel a);
/// The access level for a command category
struct Access {
/// Minimum access level required
AuthLevel auth;
/// If true only remote SNs may call the category commands
bool remote_sn;
/// If true the category requires that the local node is a SN
bool local_sn;
/// Constructor. Intentionally allows implicit conversion from an AuthLevel so that an
/// AuthLevel can be passed anywhere an Access is required (the resulting Access will have both
/// remote and local sn set to false).
Access(AuthLevel auth = AuthLevel::none, bool remote_sn = false, bool local_sn = false)
: auth{auth}, remote_sn{remote_sn}, local_sn{local_sn} {}
};
/// Simple hash implementation for a string that is *already* a hash-like value (such as a pubkey).
/// Falls back to std::hash<std::string> if given a string smaller than a size_t.
struct already_hashed {
size_t operator()(const std::string& s) const {
if (s.size() < sizeof(size_t))
return std::hash<std::string>{}(s);
size_t hash;
std::memcpy(&hash, &s[0], sizeof(hash));
return hash;
}
};
/// std::unordered_set specialization for specifying pubkeys (used, in particular, by
/// OxenMQ::set_active_sns and OxenMQ::update_active_sns); this is a std::string unordered_set that
/// also uses a specialized trivial hash function that uses part of the value itself (i.e. the
/// pubkey) directly as a hash value. (This is nice and fast for uniformly distributed values like
/// pubkeys and a terrible hash choice for anything else).
using pubkey_set = std::unordered_set<std::string, already_hashed>;
}

205
oxenmq/base32z.h Normal file
View File

@ -0,0 +1,205 @@
// Copyright (c) 2019-2021, The Oxen Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <string>
#include <string_view>
#include <array>
#include <iterator>
#include <cassert>
#include "byte_type.h"
namespace oxenmq {
namespace detail {
/// Compile-time generated lookup tables for base32z conversion. This is case insensitive (though
/// for byte -> b32z conversion we always produce lower case).
struct b32z_table {
// Store the 0-31 decoded value of every possible char; all the chars that aren't valid are set
// to 0. (If you don't trust your data, check it with is_base32z first, which uses these 0's
// to detect invalid characters -- which is why we want a full 256 element array).
char from_b32z_lut[256];
// Store the encoded character of every 0-31 (5 bit) value.
char to_b32z_lut[32];
// constexpr constructor that fills out the above (and should do it at compile time for any half
// decent compiler).
constexpr b32z_table() noexcept : from_b32z_lut{},
to_b32z_lut{
'y', 'b', 'n', 'd', 'r', 'f', 'g', '8', 'e', 'j', 'k', 'm', 'c', 'p', 'q', 'x',
'o', 't', '1', 'u', 'w', 'i', 's', 'z', 'a', '3', '4', '5', 'h', '7', '6', '9'
}
{
for (unsigned char c = 0; c < 32; c++) {
unsigned char x = to_b32z_lut[c];
from_b32z_lut[x] = c;
if (x >= 'a' && x <= 'z')
from_b32z_lut[x - 'a' + 'A'] = c;
}
}
// Convert a b32z encoded character into a 0-31 value
constexpr char from_b32z(unsigned char c) const noexcept { return from_b32z_lut[c]; }
// Convert a 0-31 value into a b32z encoded character
constexpr char to_b32z(unsigned char b) const noexcept { return to_b32z_lut[b]; }
} constexpr b32z_lut;
// This main point of this static assert is to force the compiler to compile-time build the constexpr tables.
static_assert(b32z_lut.from_b32z('w') == 20 && b32z_lut.from_b32z('T') == 17 && b32z_lut.to_b32z(5) == 'f', "");
} // namespace detail
/// Converts bytes into a base32z encoded character sequence.
template <typename InputIt, typename OutputIt>
void to_base32z(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "to_base32z requires chars/bytes");
int bits = 0; // Tracks the number of unconsumed bits held in r, will always be in [0, 4]
std::uint_fast16_t r = 0;
while (begin != end) {
r = r << 8 | static_cast<unsigned char>(*begin++);
// we just added 8 bits, so we can *always* consume 5 to produce one character, so (net) we
// are adding 3 bits.
bits += 3;
*out++ = detail::b32z_lut.to_b32z(r >> bits); // Right-shift off the bits we aren't consuming right now
// Drop the bits we don't want to keep (because we just consumed them)
r &= (1 << bits) - 1;
if (bits >= 5) { // We have enough bits to produce a second character; essentially the same as above
bits -= 5; // Except now we are just consuming 5 without having added any more
*out++ = detail::b32z_lut.to_b32z(r >> bits);
r &= (1 << bits) - 1;
}
}
if (bits > 0) // We hit the end, but still have some unconsumed bits so need one final character to append
*out++ = detail::b32z_lut.to_b32z(r << (5 - bits));
}
/// Creates a base32z string from an iterator pair of a byte sequence.
template <typename It>
std::string to_base32z(It begin, It end) {
std::string base32z;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>)
base32z.reserve((std::distance(begin, end)*8 + 4) / 5); // == bytes*8/5, rounded up.
to_base32z(begin, end, std::back_inserter(base32z));
return base32z;
}
/// Creates a base32z string from an iterable, std::string-like object
template <typename CharT>
std::string to_base32z(std::basic_string_view<CharT> s) { return to_base32z(s.begin(), s.end()); }
inline std::string to_base32z(std::string_view s) { return to_base32z<>(s); }
/// Returns true if all elements in the range are base32z characters
template <typename It>
constexpr bool is_base32z(It begin, It end) {
static_assert(sizeof(decltype(*begin)) == 1, "is_base32z requires chars/bytes");
for (; begin != end; ++begin) {
auto c = static_cast<unsigned char>(*begin);
if (detail::b32z_lut.from_b32z(c) == 0 && !(c == 'y' || c == 'Y'))
return false;
}
return true;
}
/// Returns true if all elements in the string-like value are base32z characters
template <typename CharT>
constexpr bool is_base32z(std::basic_string_view<CharT> s) { return is_base32z(s.begin(), s.end()); }
constexpr bool is_base32z(std::string_view s) { return is_base32z<>(s); }
/// Converts a sequence of base32z digits to bytes. Undefined behaviour if any characters are not
/// valid base32z alphabet characters. It is permitted for the input and output ranges to overlap
/// as long as `out` is no earlier than `begin`. Note that if you pass in a sequence that could not
/// have been created by a base32z encoding of a byte sequence, we treat the excess bits as if they
/// were not provided.
///
/// For example, "yyy" represents a 15-bit value, but a byte sequence is either 8-bit (requiring 2
/// characters) or 16-bit (requiring 4). Similarly, "yb" is an impossible encoding because it has
/// its 10th bit set (b = 00001), but a base32z encoded value should have all 0's beyond the 8th (or
/// 16th or 24th or ... bit). We treat any such bits as if they were not specified (even if they
/// are): which means "yy", "yb", "yyy", "yy9", "yd", etc. all decode to the same 1-byte value "\0".
template <typename InputIt, typename OutputIt>
void from_base32z(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "from_base32z requires chars/bytes");
uint_fast16_t curr = 0;
int bits = 0; // number of bits we've loaded into val; we always keep this < 8.
while (begin != end) {
curr = curr << 5 | detail::b32z_lut.from_b32z(static_cast<unsigned char>(*begin++));
if (bits >= 3) {
bits -= 3; // Added 5, removing 8
*out++ = static_cast<detail::byte_type_t<OutputIt>>(
static_cast<uint8_t>(curr >> bits));
curr &= (1 << bits) - 1;
} else {
bits += 5;
}
}
// Ignore any trailing bits. base32z encoding always has at least as many bits as the source
// bytes, which means we should not be able to get here from a properly encoded b32z value with
// anything other than 0s: if we have no extra bits (e.g. 5 bytes == 8 b32z chars) then we have
// a 0-bit value; if we have some extra bits (e.g. 6 bytes requires 10 b32z chars, but that
// contains 50 bits > 48 bits) then those extra bits will be 0s (and this covers the bits -= 3
// case above: it'll leave us with 0-4 extra bits, but those extra bits would be 0 if produced
// from an actual byte sequence).
//
// The "bits += 5" case, then, means that we could end with 5-7 bits. This, however, cannot be
// produced by a valid encoding:
// - 0 bytes gives us 0 chars with 0 leftover bits
// - 1 byte gives us 2 chars with 2 leftover bits
// - 2 bytes gives us 4 chars with 4 leftover bits
// - 3 bytes gives us 5 chars with 1 leftover bit
// - 4 bytes gives us 7 chars with 3 leftover bits
// - 5 bytes gives us 8 chars with 0 leftover bits (this is where the cycle repeats)
//
// So really the only way we can get 5-7 leftover bits is if you took a 0, 2 or 5 char output (or
// any 8n + {0,2,5} char output) and added a base32z character to the end. If you do that,
// well, too bad: you're giving invalid output and so we're just going to pretend that extra
// character you added isn't there by not doing anything here.
}
/// Convert a base32z sequence into a std::string of bytes. Undefined behaviour if any characters
/// are not valid (case-insensitive) base32z characters.
template <typename It>
std::string from_base32z(It begin, It end) {
std::string bytes;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>)
bytes.reserve((std::distance(begin, end)*5 + 7) / 8); // == chars*5/8, rounded up.
from_base32z(begin, end, std::back_inserter(bytes));
return bytes;
}
/// Converts base32z digits from a std::string-like object into a std::string of bytes. Undefined
/// behaviour if any characters are not valid (case-insensitive) base32z characters.
template <typename CharT>
std::string from_base32z(std::basic_string_view<CharT> s) { return from_base32z(s.begin(), s.end()); }
inline std::string from_base32z(std::string_view s) { return from_base32z<>(s); }
}

221
oxenmq/base64.h Normal file
View File

@ -0,0 +1,221 @@
// Copyright (c) 2019-2021, The Oxen Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <string>
#include <string_view>
#include <array>
#include <iterator>
#include <cassert>
#include "byte_type.h"
namespace oxenmq {
namespace detail {
/// Compile-time generated lookup tables for base64 conversion.
struct b64_table {
// Store the 0-63 decoded value of every possible char; all the chars that aren't valid are set
// to 0. (If you don't trust your data, check it with is_base64 first, which uses these 0's
// to detect invalid characters -- which is why we want a full 256 element array).
char from_b64_lut[256];
// Store the encoded character of every 0-63 (6 bit) value.
char to_b64_lut[64];
// constexpr constructor that fills out the above (and should do it at compile time for any half
// decent compiler).
constexpr b64_table() noexcept : from_b64_lut{}, to_b64_lut{} {
for (unsigned char c = 0; c < 26; c++) {
from_b64_lut[(unsigned char)('A' + c)] = 0 + c;
to_b64_lut[ (unsigned char)( 0 + c)] = 'A' + c;
}
for (unsigned char c = 0; c < 26; c++) {
from_b64_lut[(unsigned char)('a' + c)] = 26 + c;
to_b64_lut[ (unsigned char)(26 + c)] = 'a' + c;
}
for (unsigned char c = 0; c < 10; c++) {
from_b64_lut[(unsigned char)('0' + c)] = 52 + c;
to_b64_lut[ (unsigned char)(52 + c)] = '0' + c;
}
to_b64_lut[62] = '+'; from_b64_lut[(unsigned char) '+'] = 62;
to_b64_lut[63] = '/'; from_b64_lut[(unsigned char) '/'] = 63;
}
// Convert a b64 encoded character into a 0-63 value
constexpr char from_b64(unsigned char c) const noexcept { return from_b64_lut[c]; }
// Convert a 0-31 value into a b64 encoded character
constexpr char to_b64(unsigned char b) const noexcept { return to_b64_lut[b]; }
} constexpr b64_lut;
// This main point of this static assert is to force the compiler to compile-time build the constexpr tables.
static_assert(b64_lut.from_b64('/') == 63 && b64_lut.from_b64('7') == 59 && b64_lut.to_b64(38) == 'm', "");
} // namespace detail
/// Converts bytes into a base64 encoded character sequence.
template <typename InputIt, typename OutputIt>
void to_base64(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "to_base64 requires chars/bytes");
int bits = 0; // Tracks the number of unconsumed bits held in r, will always be in {0, 2, 4}
std::uint_fast16_t r = 0;
while (begin != end) {
r = r << 8 | static_cast<unsigned char>(*begin++);
// we just added 8 bits, so we can *always* consume 6 to produce one character, so (net) we
// are adding 2 bits.
bits += 2;
*out++ = detail::b64_lut.to_b64(r >> bits); // Right-shift off the bits we aren't consuming right now
// Drop the bits we don't want to keep (because we just consumed them)
r &= (1 << bits) - 1;
if (bits == 6) { // We have enough bits to produce a second character (which means we had 4 before and added 8)
bits = 0;
*out++ = detail::b64_lut.to_b64(r);
r = 0;
}
}
// If bits == 0 then we ended our 6-bit outputs coinciding with 8-bit values, i.e. at a multiple
// of 24 bits: this means we don't have anything else to output and don't need any padding.
if (bits == 2) {
// We finished with 2 unconsumed bits, which means we ended 1 byte past a 24-bit group (e.g.
// 1 byte, 4 bytes, 301 bytes, etc.); since we need to always be a multiple of 4 output
// characters that means we've produced 1: so we right-fill 0s to get the next char, then
// add two padding ='s.
*out++ = detail::b64_lut.to_b64(r << 4);
*out++ = '=';
*out++ = '=';
} else if (bits == 4) {
// 4 bits left means we produced 2 6-bit values from the first 2 bytes of a 3-byte group.
// Fill 0s to get the last one, plus one padding output.
*out++ = detail::b64_lut.to_b64(r << 2);
*out++ = '=';
}
}
/// Creates and returns a base64 string from an iterator pair of a character sequence
template <typename It>
std::string to_base64(It begin, It end) {
std::string base64;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>)
base64.reserve((std::distance(begin, end) + 2) / 3 * 4); // bytes*4/3, rounded up to the next multiple of 4
to_base64(begin, end, std::back_inserter(base64));
return base64;
}
/// Creates a base64 string from an iterable, std::string-like object
template <typename CharT>
std::string to_base64(std::basic_string_view<CharT> s) { return to_base64(s.begin(), s.end()); }
inline std::string to_base64(std::string_view s) { return to_base64<>(s); }
/// Returns true if the range is a base64 encoded value; we allow (but do not require) '=' padding,
/// but only at the end, only 1 or 2, and only if it pads out the total to a multiple of 4.
template <typename It>
constexpr bool is_base64(It begin, It end) {
static_assert(sizeof(decltype(*begin)) == 1, "is_base64 requires chars/bytes");
using std::distance;
using std::prev;
// Allow 1 or 2 padding chars *if* they pad it to a multiple of 4.
if (begin != end && distance(begin, end) % 4 == 0) {
auto last = prev(end);
if (static_cast<unsigned char>(*last) == '=')
end = last--;
if (static_cast<unsigned char>(*last) == '=')
end = last;
}
for (; begin != end; ++begin) {
auto c = static_cast<unsigned char>(*begin);
if (detail::b64_lut.from_b64(c) == 0 && c != 'A')
return false;
}
return true;
}
/// Returns true if the string-like value is a base64 encoded value
template <typename CharT>
constexpr bool is_base64(std::basic_string_view<CharT> s) { return is_base64(s.begin(), s.end()); }
constexpr bool is_base64(std::string_view s) { return is_base64(s.begin(), s.end()); }
/// Converts a sequence of base64 digits to bytes. Undefined behaviour if any characters are not
/// valid base64 alphabet characters. It is permitted for the input and output ranges to overlap as
/// long as `out` is no earlier than `begin`. Trailing padding characters are permitted but not
/// required.
///
/// It is possible to provide "impossible" base64 encoded values; for example "YWJja" which has 30
/// bits of data even though a base64 encoded byte string should have 24 (4 chars) or 36 (6 chars)
/// bits for a 3- and 4-byte input, respectively. We ignore any such "impossible" bits, and
/// similarly ignore impossible bits in the bit "overhang"; that means "YWJjZA==" (the proper
/// encoding of "abcd") and "YWJjZB", "YWJjZC", ..., "YWJjZP" all decode to the same "abcd" value:
/// the last 4 bits of the last character are essentially considered padding.
template <typename InputIt, typename OutputIt>
void from_base64(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "from_base64 requires chars/bytes");
uint_fast16_t curr = 0;
int bits = 0; // number of bits we've loaded into val; we always keep this < 8.
while (begin != end) {
auto c = static_cast<unsigned char>(*begin++);
// padding; don't bother checking if we're at the end because is_base64 is a precondition
// and we're allowed UB if it isn't satisfied.
if (c == '=') continue;
curr = curr << 6 | detail::b64_lut.from_b64(c);
if (bits == 0)
bits = 6;
else {
bits -= 2; // Added 6, removing 8
*out++ = static_cast<detail::byte_type_t<OutputIt>>(
static_cast<uint8_t>(curr >> bits));
curr &= (1 << bits) - 1;
}
}
// Don't worry about leftover bits because either they have to be 0, or they can't happen at
// all. See base32z.h for why: the reasoning is exactly the same (except using 6 bits per
// character here instead of 5).
}
/// Converts base64 digits from a iterator pair of characters into a std::string of bytes.
/// Undefined behaviour if any characters are not valid base64 characters.
template <typename It>
std::string from_base64(It begin, It end) {
std::string bytes;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>)
bytes.reserve(std::distance(begin, end)*6 / 8); // each digit carries 6 bits; this may overallocate by 1-2 bytes due to padding
from_base64(begin, end, std::back_inserter(bytes));
return bytes;
}
/// Converts base64 digits from a std::string-like object into a std::string of bytes. Undefined
/// behaviour if any characters are not valid base64 characters.
template <typename CharT>
std::string from_base64(std::basic_string_view<CharT> s) { return from_base64(s.begin(), s.end()); }
inline std::string from_base64(std::string_view s) { return from_base64<>(s); }
}

279
oxenmq/batch.h Normal file
View File

@ -0,0 +1,279 @@
// Copyright (c) 2020-2021, The Oxen Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <exception>
#include <functional>
#include <vector>
#include "oxenmq.h"
namespace oxenmq {
namespace detail {
enum class BatchState {
running, // there are still jobs to run (or running)
complete, // the batch is complete but still has a completion job to call
done // the batch is complete and has no completion function
};
struct BatchStatus {
BatchState state;
int thread;
};
// Virtual base class for Batch<R>
class Batch {
public:
// Returns the number of jobs in this batch and whether any of them are thread-specific
virtual std::pair<size_t, bool> size() const = 0;
// Returns a vector of exactly the same length of size().first containing the tagged thread ids
// of the batch jobs or 0 for general jobs.
virtual std::vector<int> threads() const = 0;
// Called in a worker thread to run the job
virtual void run_job(int i) = 0;
// Called in the main proxy thread when the worker returns from finishing a job. The return
// value tells us whether the current finishing job finishes off the batch: `running` to tell us
// there are more jobs; `complete` to tell us that the jobs are done but the completion function
// needs to be called; and `done` to signal that the jobs are done and there is no completion
// function.
virtual BatchStatus job_finished() = 0;
// Called by a worker; not scheduled until all jobs are done.
virtual void job_completion() = 0;
virtual ~Batch() = default;
};
}
/**
* Simple class that can either hold a result or an exception and retrieves the result (or raises
* the exception) via a .get() method.
*
* This is designed to be like a very stripped down version of a std::promise/std::future pair. We
* reimplemented it, however, because by ditching all the thread synchronization that promise/future
* guarantees we can substantially reduce call overhead (by a factor of ~8 according to benchmarking
* code). Since OxenMQ's proxy<->worker communication channel already gives us thread that overhead
* would just be wasted.
*
* @tparam R the value type held by the result; must be default constructible. Note, however, that
* there are specializations provided for lvalue references types and `void` (which obviously don't
* satisfy this).
*/
template <typename R, typename SFINAE = void>
class job_result {
R value;
std::exception_ptr exc;
public:
/// Sets the value. Should be called only once, or not at all if set_exception was called.
void set_value(R&& v) { value = std::move(v); }
/// Sets the exception, which will be rethrown when `get()` is called. Should be called
/// only once, or not at all if set_value() was called.
void set_exception(std::exception_ptr e) { exc = std::move(e); }
/// Retrieves the value. If an exception was set instead of a value then that exception is
/// thrown instead. Note that the interval value is moved out of the held value so you should
/// not call this multiple times.
R get() {
if (exc) std::rethrow_exception(exc);
return std::move(value);
}
};
/** job_result specialization for reference types */
template <typename R>
class job_result<R, std::enable_if_t<std::is_lvalue_reference<R>::value>> {
std::remove_reference_t<R>* value_ptr;
std::exception_ptr exc;
public:
void set_value(R v) { value_ptr = &v; }
void set_exception(std::exception_ptr e) { exc = std::move(e); }
R get() {
if (exc) std::rethrow_exception(exc);
return *value_ptr;
}
};
/** job_result specialization for void; there is no value, but exceptions are still captured
* (rethrown when `get()` is called).
*/
template<>
class job_result<void> {
std::exception_ptr exc;
public:
void set_exception(std::exception_ptr e) { exc = std::move(e); }
// Returns nothing, but rethrows if there is a captured exception.
void get() { if (exc) std::rethrow_exception(exc); }
};
/// Helper class used to set up batches of jobs to be scheduled via the oxenmq job handler.
///
/// @tparam R - the return type of the individual jobs
///
template <typename R>
class Batch final : private detail::Batch {
friend class OxenMQ;
public:
/// The completion function type, called after all jobs have finished.
using CompletionFunc = std::function<void(std::vector<job_result<R>> results)>;
// Default constructor
Batch() = default;
// movable
Batch(Batch&&) = default;
Batch &operator=(Batch&&) = default;
// non-copyable
Batch(const Batch&) = delete;
Batch &operator=(const Batch&) = delete;
private:
std::vector<std::pair<std::function<R()>, int>> jobs;
std::vector<job_result<R>> results;
CompletionFunc complete;
std::size_t jobs_outstanding = 0;
int complete_in_thread = 0;
bool started = false;
bool tagged_thread_jobs = false;
void check_not_started() {
if (started)
throw std::logic_error("Cannot add jobs or completion function after starting a oxenmq::Batch!");
}
public:
/// Preallocates space in the internal vector that stores jobs.
void reserve(std::size_t num) {
jobs.reserve(num);
results.reserve(num);
}
/// Adds a job. This takes any callable object that is invoked with no arguments and returns R
/// (the Batch return type). The tasks will be scheduled and run when the next worker thread is
/// available. The called function may throw exceptions (which will be propagated to the
/// completion function through the job_result values). There is no guarantee on the order of
/// invocation of the jobs.
///
/// \param job the callback
/// \param thread an optional TaggedThreadID indicating a thread in which this job must run
void add_job(std::function<R()> job, std::optional<TaggedThreadID> thread = std::nullopt) {
check_not_started();
if (thread && thread->_id == -1)
// There are some special case internal jobs where we allow this, but they use the
// private method below that doesn't have this check.
throw std::logic_error{"Cannot add a proxy thread batch job -- this makes no sense"};
add_job(std::move(job), thread ? thread->_id : 0);
}
/// Sets the completion function to invoke after all jobs have finished. If this is not set
/// then jobs simply run and results are discarded.
///
/// \param comp - function to call when all jobs have finished
/// \param thread - optional tagged thread in which to schedule the completion job. If not
/// provided then the completion job is scheduled in the pool of batch job threads.
///
/// `thread` can be provided the value &OxenMQ::run_in_proxy to invoke the completion function
/// *IN THE PROXY THREAD* itself after all jobs have finished. Be very, very careful: this
/// should be a nearly trivial job that does not require any substantial CPU time and does not
/// block for any reason. This is only intended for the case where the completion job is so
/// trivial that it will take less time than simply queuing the job to be executed by another
/// thread.
void completion(CompletionFunc comp, std::optional<TaggedThreadID> thread = std::nullopt) {
check_not_started();
if (complete)
throw std::logic_error("Completion function can only be set once");
complete = std::move(comp);
complete_in_thread = thread ? thread->_id : 0;
}
private:
void add_job(std::function<R()> job, int thread_id) {
jobs.emplace_back(std::move(job), thread_id);
results.emplace_back();
jobs_outstanding++;
if (thread_id != 0)
tagged_thread_jobs = true;
}
std::pair<std::size_t, bool> size() const override {
return {jobs.size(), tagged_thread_jobs};
}
std::vector<int> threads() const override {
std::vector<int> t;
t.reserve(jobs.size());
for (auto& j : jobs)
t.push_back(j.second);
return t;
};
template <typename S = R>
void set_value(job_result<S>& r, std::function<S()>& f) { r.set_value(f()); }
void set_value(job_result<void>&, std::function<void()>& f) { f(); }
void run_job(const int i) override {
// called by worker thread
auto& r = results[i];
try {
set_value(r, jobs[i].first);
} catch (...) {
r.set_exception(std::current_exception());
}
}
detail::BatchStatus job_finished() override {
--jobs_outstanding;
if (jobs_outstanding)
return {detail::BatchState::running, 0};
if (complete)
return {detail::BatchState::complete, complete_in_thread};
return {detail::BatchState::done, 0};
}
void job_completion() override {
return complete(std::move(results));
}
};
template <typename R>
void OxenMQ::batch(Batch<R>&& batch) {
if (batch.size().first == 0)
throw std::logic_error("Cannot batch a a job batch with 0 jobs");
// Need to send this over to the proxy thread via the base class pointer. It assumes ownership.
auto* baseptr = static_cast<detail::Batch*>(new Batch<R>(std::move(batch)));
detail::send_control(get_control_socket(), "BATCH", bt_serialize(reinterpret_cast<uintptr_t>(baseptr)));
}
}

6
lokimq/bt_serialize.cpp → oxenmq/bt_serialize.cpp
View File

@ -1,4 +1,4 @@
// Copyright (c) 2019-2020, The Loki Project
// Copyright (c) 2019-2021, The Oxen Project
//
// All rights reserved.
//
@ -29,7 +29,7 @@
#include "bt_serialize.h"
#include <iterator>
namespace lokimq {
namespace oxenmq {
namespace detail {
/// Reads digits into an unsigned 64-bit int.
@ -228,4 +228,4 @@ std::pair<std::string_view, std::string_view> bt_dict_consumer::next_string() {
}
} // namespace lokimq
} // namespace oxenmq

915
oxenmq/bt_serialize.h Normal file
View File

@ -0,0 +1,915 @@
// Copyright (c) 2019-2020, The Oxen Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <vector>
#include <functional>
#include <cstring>
#include <sstream>
#include <ostream>
#include <string>
#include <string_view>
#include "variant.h"
#include <cstdint>
#include <limits>
#include <stdexcept>
#include <type_traits>
#include <utility>
#include <tuple>
#include <algorithm>
#include "bt_value.h"
namespace oxenmq {
using namespace std::literals;
/** \file
* OxenMQ serialization for internal commands is very simple: we support two primitive types,
* strings and integers, and two container types, lists and dicts with string keys. On the wire
* these go in BitTorrent byte encoding as described in BEP-0003
* (https://www.bittorrent.org/beps/bep_0003.html#bencoding).
*
* On the C++ side, on input we allow strings, integral types, STL-like containers of these types,
* and STL-like containers of pairs with a string first value and any of these types as second
* value. We also accept std::variants of these.
*
* One minor deviation from BEP-0003 is that we don't support serializing values that don't fit in a
* 64-bit integer (BEP-0003 specifies arbitrary precision integers).
*
* On deserialization we can either deserialize into a special bt_value type supports everything
* (with arbitrary nesting), or we can fill a container of your given type (though this fails if the
* container isn't compatible with the deserialized data).
*
* There is also a stream deserialization that allows you to deserialize without needing heap
* allocations (as long as you know the precise data structure layout).
*/
/// Exception throw if deserialization fails
class bt_deserialize_invalid : public std::invalid_argument {
using std::invalid_argument::invalid_argument;
};
/// A more specific subclass that is thown if the serialization type is an initial mismatch: for
/// example, trying deserializing an int but the next thing in input is a list. This is not,
/// however, thrown if the type initially looks fine but, say, a nested serialization fails. This
/// error will only be thrown when the input stream has not been advanced (and so can be tried for a
/// different type).
class bt_deserialize_invalid_type : public bt_deserialize_invalid {
using bt_deserialize_invalid::bt_deserialize_invalid;
};
namespace detail {
/// Reads digits into an unsigned 64-bit int.
uint64_t extract_unsigned(std::string_view& s);
// (Provide non-constant lvalue and rvalue ref functions so that we only accept explicit
// string_views but not implicitly converted ones)
inline uint64_t extract_unsigned(std::string_view&& s) { return extract_unsigned(s); }
// Fallback base case; we only get here if none of the partial specializations below work
template <typename T, typename SFINAE = void>
struct bt_serialize { static_assert(!std::is_same_v<T, T>, "Cannot serialize T: unsupported type for bt serialization"); };
template <typename T, typename SFINAE = void>
struct bt_deserialize { static_assert(!std::is_same_v<T, T>, "Cannot deserialize T: unsupported type for bt deserialization"); };
/// Checks that we aren't at the end of a string view and throws if we are.
inline void bt_need_more(const std::string_view &s) {
if (s.empty())
throw bt_deserialize_invalid{"Unexpected end of string while deserializing"};
}
/// Deserializes a signed or unsigned 64-bit integer from a string. Sets the second bool to true
/// iff the value read was negative, false if positive; in either case the unsigned value is return
/// in .first. Throws an exception if the read value doesn't fit in a int64_t (if negative) or a
/// uint64_t (if positive). Removes consumed characters from the string_view.
std::pair<uint64_t, bool> bt_deserialize_integer(std::string_view& s);
/// Integer specializations
template <typename T>
struct bt_serialize<T, std::enable_if_t<std::is_integral_v<T>>> {
static_assert(sizeof(T) <= sizeof(uint64_t), "Serialization of integers larger than uint64_t is not supported");
void operator()(std::ostream &os, const T &val) {
// Cast 1-byte types to a larger type to avoid iostream interpreting them as single characters
using output_type = std::conditional_t<(sizeof(T) > 1), T, std::conditional_t<std::is_signed_v<T>, int, unsigned>>;
os << 'i' << static_cast<output_type>(val) << 'e';
}
};
template <typename T>
struct bt_deserialize<T, std::enable_if_t<std::is_integral_v<T>>> {
void operator()(std::string_view& s, T &val) {
constexpr uint64_t umax = static_cast<uint64_t>(std::numeric_limits<T>::max());
constexpr int64_t smin = static_cast<int64_t>(std::numeric_limits<T>::min());
auto [magnitude, negative] = bt_deserialize_integer(s);
if (std::is_signed_v<T>) {
if (!negative) {
if (magnitude > umax)
throw bt_deserialize_invalid("Integer deserialization failed: found too-large value " + std::to_string(magnitude) + " > " + std::to_string(umax));
val = static_cast<T>(magnitude);
} else {
auto sval = -static_cast<int64_t>(magnitude);
if (!std::is_same_v<T, int64_t> && sval < smin)
throw bt_deserialize_invalid("Integer deserialization failed: found too-low value " + std::to_string(sval) + " < " + std::to_string(smin));
val = static_cast<T>(sval);
}
} else {
if (negative)
throw bt_deserialize_invalid("Integer deserialization failed: found negative value -" + std::to_string(magnitude) + " but type is unsigned");
if (!std::is_same_v<T, uint64_t> && magnitude > umax)
throw bt_deserialize_invalid("Integer deserialization failed: found too-large value " + std::to_string(magnitude) + " > " + std::to_string(umax));
val = static_cast<T>(magnitude);
}
}
};
extern template struct bt_deserialize<int64_t>;
extern template struct bt_deserialize<uint64_t>;
template <>
struct bt_serialize<std::string_view> {
void operator()(std::ostream &os, const std::string_view &val) { os << val.size(); os.put(':'); os.write(val.data(), val.size()); }
};
template <>
struct bt_deserialize<std::string_view> {
void operator()(std::string_view& s, std::string_view& val);
};
/// String specialization
template <>
struct bt_serialize<std::string> {
void operator()(std::ostream &os, const std::string &val) { bt_serialize<std::string_view>{}(os, val); }
};
template <>
struct bt_deserialize<std::string> {
void operator()(std::string_view& s, std::string& val) { std::string_view view; bt_deserialize<std::string_view>{}(s, view); val = {view.data(), view.size()}; }
};
/// char * and string literals -- we allow serialization for convenience, but not deserialization
template <>
struct bt_serialize<char *> {
void operator()(std::ostream &os, const char *str) { bt_serialize<std::string_view>{}(os, {str, std::strlen(str)}); }
};
template <size_t N>
struct bt_serialize<char[N]> {
void operator()(std::ostream &os, const char *str) { bt_serialize<std::string_view>{}(os, {str, N-1}); }
};
/// Partial dict validity; we don't check the second type for serializability, that will be handled
/// via the base case static_assert if invalid.
template <typename T, typename = void> struct is_bt_input_dict_container_impl : std::false_type {};
template <typename T>
struct is_bt_input_dict_container_impl<T, std::enable_if_t<
std::is_same_v<std::string, std::remove_cv_t<typename T::value_type::first_type>> ||
std::is_same_v<std::string_view, std::remove_cv_t<typename T::value_type::first_type>>,
std::void_t<typename T::const_iterator /* is const iterable */,
typename T::value_type::second_type /* has a second type */>>>
: std::true_type {};
/// Determines whether the type looks like something we can insert into (using `v.insert(v.end(), x)`)
template <typename T, typename = void> struct is_bt_insertable_impl : std::false_type {};
template <typename T>
struct is_bt_insertable_impl<T,
std::void_t<decltype(std::declval<T>().insert(std::declval<T>().end(), std::declval<typename T::value_type>()))>>
: std::true_type {};
template <typename T>
constexpr bool is_bt_insertable = is_bt_insertable_impl<T>::value;
/// Determines whether the given type looks like a compatible map (i.e. has std::string keys) that
/// we can insert into.
template <typename T, typename = void> struct is_bt_output_dict_container_impl : std::false_type {};
template <typename T>
struct is_bt_output_dict_container_impl<T, std::enable_if_t<
std::is_same_v<std::string, std::remove_cv_t<typename T::value_type::first_type>> && is_bt_insertable<T>,
std::void_t<typename T::value_type::second_type /* has a second type */>>>
: std::true_type {};
template <typename T>
constexpr bool is_bt_output_dict_container = is_bt_output_dict_container_impl<T>::value;
template <typename T>
constexpr bool is_bt_input_dict_container = is_bt_output_dict_container_impl<T>::value;
// Sanity checks:
static_assert(is_bt_input_dict_container<bt_dict>);
static_assert(is_bt_output_dict_container<bt_dict>);
/// Specialization for a dict-like container (such as an unordered_map). We accept anything for a
/// dict that is const iterable over something that looks like a pair with std::string for first
/// value type. The value (i.e. second element of the pair) also must be serializable.
template <typename T>
struct bt_serialize<T, std::enable_if_t<is_bt_input_dict_container<T>>> {
using second_type = typename T::value_type::second_type;
using ref_pair = std::reference_wrapper<const typename T::value_type>;
void operator()(std::ostream &os, const T &dict) {
os << 'd';
std::vector<ref_pair> pairs;
pairs.reserve(dict.size());
for (const auto &pair : dict)
pairs.emplace(pairs.end(), pair);
std::sort(pairs.begin(), pairs.end(), [](ref_pair a, ref_pair b) { return a.get().first < b.get().first; });
for (auto &ref : pairs) {
bt_serialize<std::string>{}(os, ref.get().first);
bt_serialize<second_type>{}(os, ref.get().second);
}
os << 'e';
}
};
template <typename T>
struct bt_deserialize<T, std::enable_if_t<is_bt_output_dict_container<T>>> {
using second_type = typename T::value_type::second_type;
void operator()(std::string_view& s, T& dict) {
// Smallest dict is 2 bytes "de", for an empty dict.
if (s.size() < 2) throw bt_deserialize_invalid("Deserialization failed: end of string found where dict expected");
if (s[0] != 'd') throw bt_deserialize_invalid_type("Deserialization failed: expected 'd', found '"s + s[0] + "'"s);
s.remove_prefix(1);
dict.clear();
bt_deserialize<std::string> key_deserializer;
bt_deserialize<second_type> val_deserializer;
while (!s.empty() && s[0] != 'e') {
std::string key;
second_type val;
key_deserializer(s, key);
val_deserializer(s, val);
dict.insert(dict.end(), typename T::value_type{std::move(key), std::move(val)});
}
if (s.empty())
throw bt_deserialize_invalid("Deserialization failed: encountered end of string before dict was finished");
s.remove_prefix(1); // Consume the 'e'
}
};
/// Accept anything that looks iterable; value serialization validity isn't checked here (it fails
/// via the base case static assert).
template <typename T, typename = void> struct is_bt_input_list_container_impl : std::false_type {};
template <typename T>
struct is_bt_input_list_container_impl<T, std::enable_if_t<
!std::is_same_v<T, std::string> && !std::is_same_v<T, std::string_view> && !is_bt_input_dict_container<T>,
std::void_t<typename T::const_iterator, typename T::value_type>>>
: std::true_type {};
template <typename T, typename = void> struct is_bt_output_list_container_impl : std::false_type {};
template <typename T>
struct is_bt_output_list_container_impl<T, std::enable_if_t<
!std::is_same_v<T, std::string> && !is_bt_output_dict_container<T> && is_bt_insertable<T>>>
: std::true_type {};
template <typename T>
constexpr bool is_bt_output_list_container = is_bt_output_list_container_impl<T>::value;
template <typename T>
constexpr bool is_bt_input_list_container = is_bt_input_list_container_impl<T>::value;
// Sanity checks:
static_assert(is_bt_input_list_container<bt_list>);
static_assert(is_bt_output_list_container<bt_list>);
/// List specialization
template <typename T>
struct bt_serialize<T, std::enable_if_t<is_bt_input_list_container<T>>> {
void operator()(std::ostream& os, const T& list) {
os << 'l';
for (const auto &v : list)
bt_serialize<std::remove_cv_t<typename T::value_type>>{}(os, v);
os << 'e';
}
};
template <typename T>
struct bt_deserialize<T, std::enable_if_t<is_bt_output_list_container<T>>> {
using value_type = typename T::value_type;
void operator()(std::string_view& s, T& list) {
// Smallest list is 2 bytes "le", for an empty list.
if (s.size() < 2) throw bt_deserialize_invalid("Deserialization failed: end of string found where list expected");
if (s[0] != 'l') throw bt_deserialize_invalid_type("Deserialization failed: expected 'l', found '"s + s[0] + "'"s);
s.remove_prefix(1);
list.clear();
bt_deserialize<value_type> deserializer;
while (!s.empty() && s[0] != 'e') {
value_type v;
deserializer(s, v);
list.insert(list.end(), std::move(v));
}
if (s.empty())
throw bt_deserialize_invalid("Deserialization failed: encountered end of string before list was finished");
s.remove_prefix(1); // Consume the 'e'
}
};
/// Serializes a tuple or pair of serializable values (as a list on the wire)
/// Common implementation for both tuple and pair:
template <template<typename...> typename Tuple, typename... T>
struct bt_serialize_tuple {
private:
template <size_t... Is>
void operator()(std::ostream& os, const Tuple<T...>& elems, std::index_sequence<Is...>) {
os << 'l';
(bt_serialize<T>{}(os, std::get<Is>(elems)), ...);
os << 'e';
}
public:
void operator()(std::ostream& os, const Tuple<T...>& elems) {
operator()(os, elems, std::index_sequence_for<T...>{});
}
};
template <template<typename...> typename Tuple, typename... T>
struct bt_deserialize_tuple {
private:
template <size_t... Is>
void operator()(std::string_view& s, Tuple<T...>& elems, std::index_sequence<Is...>) {
// Smallest list is 2 bytes "le", for an empty list.
if (s.size() < 2) throw bt_deserialize_invalid("Deserialization failed: end of string found where tuple expected");
if (s[0] != 'l') throw bt_deserialize_invalid_type("Deserialization of tuple failed: expected 'l', found '"s + s[0] + "'"s);
s.remove_prefix(1);
(bt_deserialize<T>{}(s, std::get<Is>(elems)), ...);
if (s.empty())
throw bt_deserialize_invalid("Deserialization failed: encountered end of string before tuple was finished");
if (s[0] != 'e')
throw bt_deserialize_invalid("Deserialization failed: expected end of tuple but found something else");
s.remove_prefix(1); // Consume the 'e'
}
public:
void operator()(std::string_view& s, Tuple<T...>& elems) {
operator()(s, elems, std::index_sequence_for<T...>{});
}
};
template <typename... T>
struct bt_serialize<std::tuple<T...>> : bt_serialize_tuple<std::tuple, T...> {};
template <typename... T>
struct bt_deserialize<std::tuple<T...>> : bt_deserialize_tuple<std::tuple, T...> {};
template <typename S, typename T>
struct bt_serialize<std::pair<S, T>> : bt_serialize_tuple<std::pair, S, T> {};
template <typename S, typename T>
struct bt_deserialize<std::pair<S, T>> : bt_deserialize_tuple<std::pair, S, T> {};
template <typename T>
constexpr bool is_bt_tuple = false;
template <typename... T>
constexpr bool is_bt_tuple<std::tuple<T...>> = true;
template <typename S, typename T>
constexpr bool is_bt_tuple<std::pair<S, T>> = true;
template <typename T>
constexpr bool is_bt_deserializable = std::is_same_v<T, std::string> || std::is_integral_v<T> ||
is_bt_output_dict_container<T> || is_bt_output_list_container<T> || is_bt_tuple<T>;
// General template and base case; this base will only actually be invoked when Ts... is empty,
// which means we reached the end without finding any variant type capable of holding the value.
template <typename SFINAE, typename Variant, typename... Ts>
struct bt_deserialize_try_variant_impl {
void operator()(std::string_view&, Variant&) {
throw bt_deserialize_invalid("Deserialization failed: could not deserialize value into any variant type");
}
};
template <typename... Ts, typename Variant>
void bt_deserialize_try_variant(std::string_view& s, Variant& variant) {
bt_deserialize_try_variant_impl<void, Variant, Ts...>{}(s, variant);
}
template <typename Variant, typename T, typename... Ts>
struct bt_deserialize_try_variant_impl<std::enable_if_t<is_bt_deserializable<T>>, Variant, T, Ts...> {
void operator()(std::string_view& s, Variant& variant) {
if ( is_bt_output_list_container<T> ? s[0] == 'l' :
is_bt_tuple<T> ? s[0] == 'l' :
is_bt_output_dict_container<T> ? s[0] == 'd' :
std::is_integral_v<T> ? s[0] == 'i' :
std::is_same_v<T, std::string> ? s[0] >= '0' && s[0] <= '9' :
false) {
T val;
bt_deserialize<T>{}(s, val);
variant = std::move(val);
} else {
bt_deserialize_try_variant<Ts...>(s, variant);
}
}
};
template <typename Variant, typename T, typename... Ts>
struct bt_deserialize_try_variant_impl<std::enable_if_t<!is_bt_deserializable<T>>, Variant, T, Ts...> {
void operator()(std::string_view& s, Variant& variant) {
// Unsupported deserialization type, skip it
bt_deserialize_try_variant<Ts...>(s, variant);
}
};
// Serialization of a variant; all variant types must be bt-serializable.
template <typename... Ts>
struct bt_serialize<std::variant<Ts...>, std::void_t<bt_serialize<Ts>...>> {
void operator()(std::ostream& os, const std::variant<Ts...>& val) {
var::visit(
[&os] (const auto& val) {
using T = std::remove_cv_t<std::remove_reference_t<decltype(val)>>;
bt_serialize<T>{}(os, val);
},
val);
}
};
// Deserialization to a variant; at least one variant type must be bt-deserializble.
template <typename... Ts>
struct bt_deserialize<std::variant<Ts...>, std::enable_if_t<(is_bt_deserializable<Ts> || ...)>> {
void operator()(std::string_view& s, std::variant<Ts...>& val) {
bt_deserialize_try_variant<Ts...>(s, val);
}
};
template <>
struct bt_serialize<bt_value> : bt_serialize<bt_variant> {};
template <>
struct bt_deserialize<bt_value> {
void operator()(std::string_view& s, bt_value& val);
};
template <typename T>
struct bt_stream_serializer {
const T &val;
explicit bt_stream_serializer(const T &val) : val{val} {}
operator std::string() const {
std::ostringstream oss;
oss << *this;
return oss.str();
}
};
template <typename T>
std::ostream &operator<<(std::ostream &os, const bt_stream_serializer<T> &s) {
bt_serialize<T>{}(os, s.val);
return os;
}
} // namespace detail
/// Returns a wrapper around a value reference that can serialize the value directly to an output
/// stream. This class is intended to be used inline (i.e. without being stored) as in:
///
/// std::list<int> my_list{{1,2,3}};
/// std::cout << bt_serializer(my_list);
///
/// While it is possible to store the returned object and use it, such as:
///
/// auto encoded = bt_serializer(42);
/// std::cout << encoded;
///
/// this approach is not generally recommended: the returned object stores a reference to the
/// passed-in type, which may not survive. If doing this note that it is the caller's
/// responsibility to ensure the serializer is not used past the end of the lifetime of the value
/// being serialized.
///
/// Also note that serializing directly to an output stream is more efficient as no intermediate
/// string containing the entire serialization has to be constructed.
///
template <typename T>
detail::bt_stream_serializer<T> bt_serializer(const T &val) { return detail::bt_stream_serializer<T>{val}; }
/// Serializes the given value into a std::string.
///
/// int number = 42;
/// std::string encoded = bt_serialize(number);
/// // Equivalent:
/// //auto encoded = (std::string) bt_serialize(number);
///
/// This takes any serializable type: integral types, strings, lists of serializable types, and
/// string->value maps of serializable types.
template <typename T>
std::string bt_serialize(const T &val) { return bt_serializer(val); }
/// Deserializes the given string view directly into `val`. Usage:
///
/// std::string encoded = "i42e";
/// int value;
/// bt_deserialize(encoded, value); // Sets value to 42
///
template <typename T, std::enable_if_t<!std::is_const_v<T>, int> = 0>
void bt_deserialize(std::string_view s, T& val) {
return detail::bt_deserialize<T>{}(s, val);
}
/// Deserializes the given string_view into a `T`, which is returned.
///
/// std::string encoded = "li1ei2ei3ee"; // bt-encoded list of ints: [1,2,3]
/// auto mylist = bt_deserialize<std::list<int>>(encoded);
///
template <typename T>
T bt_deserialize(std::string_view s) {
T val;
bt_deserialize(s, val);
return val;
}
/// Deserializes the given value into a generic `bt_value` type (wrapped std::variant) which is
/// capable of holding all possible BT-encoded values (including recursion).
///
/// Example:
///
/// std::string encoded = "i42e";
/// auto val = bt_get(encoded);
/// int v = get_int<int>(val); // fails unless the encoded value was actually an integer that
/// // fits into an `int`
///
inline bt_value bt_get(std::string_view s) {
return bt_deserialize<bt_value>(s);
}
/// Helper functions to extract a value of some integral type from a bt_value which contains either
/// a int64_t or uint64_t. Does range checking, throwing std::overflow_error if the stored value is
/// outside the range of the target type.
///
/// Example:
///
/// std::string encoded = "i123456789e";
/// auto val = bt_get(encoded);
/// auto v = get_int<uint32_t>(val); // throws if the decoded value doesn't fit in a uint32_t
template <typename IntType, std::enable_if_t<std::is_integral_v<IntType>, int> = 0>
IntType get_int(const bt_value &v) {
if (auto* value = std::get_if<uint64_t>(&v)) {
if constexpr (!std::is_same_v<IntType, uint64_t>)
if (*value > static_cast<uint64_t>(std::numeric_limits<IntType>::max()))
throw std::overflow_error("Unable to extract integer value: stored value is too large for the requested type");
return static_cast<IntType>(*value);
}
int64_t value = var::get<int64_t>(v); // throws if no int contained
if constexpr (!std::is_same_v<IntType, int64_t>)
if (value > static_cast<int64_t>(std::numeric_limits<IntType>::max())
|| value < static_cast<int64_t>(std::numeric_limits<IntType>::min()))
throw std::overflow_error("Unable to extract integer value: stored value is outside the range of the requested type");
return static_cast<IntType>(value);
}
namespace detail {
template <typename Tuple, size_t... Is>
void get_tuple_impl(Tuple& t, const bt_list& l, std::index_sequence<Is...>);
}
/// Converts a bt_list into the given template std::tuple or std::pair. Throws a
/// std::invalid_argument if the list has the wrong size or wrong element types. Supports recursion
/// (i.e. if the tuple itself contains tuples or pairs). The tuple (or nested tuples) may only
/// contain integral types, strings, string_views, bt_list, bt_dict, and tuples/pairs of those.
template <typename Tuple>
Tuple get_tuple(const bt_list& x) {
Tuple t;
detail::get_tuple_impl(t, x, std::make_index_sequence<std::tuple_size_v<Tuple>>{});
return t;
}
template <typename Tuple>
Tuple get_tuple(const bt_value& x) {
return get_tuple<Tuple>(var::get<bt_list>(static_cast<const bt_variant&>(x)));
}
namespace detail {
template <typename T, typename It>
void get_tuple_impl_one(T& t, It& it) {
const bt_variant& v = *it++;
if constexpr (std::is_integral_v<T>) {
t = oxenmq::get_int<T>(v);
} else if constexpr (is_bt_tuple<T>) {
if (std::holds_alternative<bt_list>(v))
throw std::invalid_argument{"Unable to convert tuple: cannot create sub-tuple from non-bt_list"};
t = get_tuple<T>(var::get<bt_list>(v));
} else if constexpr (std::is_same_v<std::string, T> || std::is_same_v<std::string_view, T>) {
// If we request a string/string_view, we might have the other one and need to copy/view it.
if (std::holds_alternative<std::string_view>(v))
t = var::get<std::string_view>(v);
else
t = var::get<std::string>(v);
} else {
t = var::get<T>(v);
}
}
template <typename Tuple, size_t... Is>
void get_tuple_impl(Tuple& t, const bt_list& l, std::index_sequence<Is...>) {
if (l.size() != sizeof...(Is))
throw std::invalid_argument{"Unable to convert tuple: bt_list has wrong size"};
auto it = l.begin();
(get_tuple_impl_one(std::get<Is>(t), it), ...);
}
} // namespace detail
/// Class that allows you to walk through a bt-encoded list in memory without copying or allocating
/// memory. It accesses existing memory directly and so the caller must ensure that the referenced
/// memory stays valid for the lifetime of the bt_list_consumer object.
class bt_list_consumer {
protected:
std::string_view data;
bt_list_consumer() = default;
public:
bt_list_consumer(std::string_view data_);
/// Copy constructor. Making a copy copies the current position so can be used for multipass
/// iteration through a list.
bt_list_consumer(const bt_list_consumer&) = default;
bt_list_consumer& operator=(const bt_list_consumer&) = default;
/// Get a copy of the current buffer
std::string_view current_buffer() const { return data; }
/// Returns true if the next value indicates the end of the list
bool is_finished() const { return data.front() == 'e'; }
/// Returns true if the next element looks like an encoded string
bool is_string() const { return data.front() >= '0' && data.front() <= '9'; }
/// Returns true if the next element looks like an encoded integer
bool is_integer() const { return data.front() == 'i'; }
/// Returns true if the next element looks like an encoded negative integer
bool is_negative_integer() const { return is_integer() && data.size() >= 2 && data[1] == '-'; }
/// Returns true if the next element looks like an encoded list
bool is_list() const { return data.front() == 'l'; }
/// Returns true if the next element looks like an encoded dict
bool is_dict() const { return data.front() == 'd'; }
/// Attempt to parse the next value as a string (and advance just past it). Throws if the next
/// value is not a string.
std::string consume_string();
std::string_view consume_string_view();
/// Attempts to parse the next value as an integer (and advance just past it). Throws if the
/// next value is not an integer.
template <typename IntType>
IntType consume_integer() {
if (!is_integer()) throw bt_deserialize_invalid_type{"next value is not an integer"};
std::string_view next{data};
IntType ret;
detail::bt_deserialize<IntType>{}(next, ret);
data = next;
return ret;
}
/// Consumes a list, return it as a list-like type. Can also be used for tuples/pairs. This
/// typically requires dynamic allocation, but only has to parse the data once. Compare with
/// consume_list_data() which allows alloc-free traversal, but requires parsing twice (if the
/// contents are to be used).
template <typename T = bt_list>
T consume_list() {
T list;
consume_list(list);
return list;
}
/// Same as above, but takes a pre-existing list-like data type.
template <typename T>
void consume_list(T& list) {
if (!is_list()) throw bt_deserialize_invalid_type{"next bt value is not a list"};
std::string_view n{data};
detail::bt_deserialize<T>{}(n, list);
data = n;
}
/// Consumes a dict, return it as a dict-like type. This typically requires dynamic allocation,
/// but only has to parse the data once. Compare with consume_dict_data() which allows
/// alloc-free traversal, but requires parsing twice (if the contents are to be used).
template <typename T = bt_dict>
T consume_dict() {
T dict;
consume_dict(dict);
return dict;
}
/// Same as above, but takes a pre-existing dict-like data type.
template <typename T>
void consume_dict(T& dict) {
if (!is_dict()) throw bt_deserialize_invalid_type{"next bt value is not a dict"};
std::string_view n{data};
detail::bt_deserialize<T>{}(n, dict);
data = n;
}
/// Consumes a value without returning it.
void skip_value();
/// Attempts to parse the next value as a list and returns the string_view that contains the
/// entire thing. This is recursive into both lists and dicts and likely to be quite
/// inefficient for large, nested structures (unless the values only need to be skipped but
/// aren't separately needed). This, however, does not require dynamic memory allocation.
std::string_view consume_list_data();
/// Attempts to parse the next value as a dict and returns the string_view that contains the
/// entire thing. This is recursive into both lists and dicts and likely to be quite
/// inefficient for large, nested structures (unless the values only need to be skipped but
/// aren't separately needed). This, however, does not require dynamic memory allocation.
std::string_view consume_dict_data();
};
/// Class that allows you to walk through key-value pairs of a bt-encoded dict in memory without
/// copying or allocating memory. It accesses existing memory directly and so the caller must
/// ensure that the referenced memory stays valid for the lifetime of the bt_dict_consumer object.
class bt_dict_consumer : private bt_list_consumer {
std::string_view key_;
/// Consume the key if not already consumed and there is a key present (rather than 'e').
/// Throws exception if what should be a key isn't a string, or if the key consumes the entire
/// data (i.e. requires that it be followed by something). Returns true if the key was consumed
/// (either now or previously and cached).
bool consume_key();
/// Clears the cached key and returns it. Must have already called consume_key directly or
/// indirectly via one of the `is_{...}` methods.
std::string_view flush_key() {
std::string_view k;
k.swap(key_);
return k;
}
public:
bt_dict_consumer(std::string_view data_);
/// Copy constructor. Making a copy copies the current position so can be used for multipass
/// iteration through a list.
bt_dict_consumer(const bt_dict_consumer&) = default;
bt_dict_consumer& operator=(const bt_dict_consumer&) = default;
/// Returns true if the next value indicates the end of the dict
bool is_finished() { return !consume_key() && data.front() == 'e'; }
/// Operator bool is an alias for `!is_finished()`
operator bool() { return !is_finished(); }
/// Returns true if the next value looks like an encoded string
bool is_string() { return consume_key() && data.front() >= '0' && data.front() <= '9'; }
/// Returns true if the next element looks like an encoded integer
bool is_integer() { return consume_key() && data.front() == 'i'; }
/// Returns true if the next element looks like an encoded negative integer
bool is_negative_integer() { return is_integer() && data.size() >= 2 && data[1] == '-'; }
/// Returns true if the next element looks like an encoded list
bool is_list() { return consume_key() && data.front() == 'l'; }
/// Returns true if the next element looks like an encoded dict
bool is_dict() { return consume_key() && data.front() == 'd'; }
/// Returns the key of the next pair. This does not have to be called; it is also returned by
/// all of the other consume_* methods. The value is cached whether called here or by some
/// other method; accessing it multiple times simple accesses the cache until the next value is
/// consumed.
std::string_view key() {
if (!consume_key())
throw bt_deserialize_invalid{"Cannot access next key: at the end of the dict"};
return key_;
}
/// Attempt to parse the next value as a string->string pair (and advance just past it). Throws
/// if the next value is not a string.
std::pair<std::string_view, std::string_view> next_string();
/// Attempts to parse the next value as an string->integer pair (and advance just past it).
/// Throws if the next value is not an integer.
template <typename IntType>
std::pair<std::string_view, IntType> next_integer() {
if (!is_integer()) throw bt_deserialize_invalid_type{"next bt dict value is not an integer"};
std::pair<std::string_view, IntType> ret;
ret.second = bt_list_consumer::consume_integer<IntType>();
ret.first = flush_key();
return ret;
}
/// Consumes a string->list pair, return it as a list-like type. This typically requires
/// dynamic allocation, but only has to parse the data once. Compare with consume_list_data()
/// which allows alloc-free traversal, but requires parsing twice (if the contents are to be
/// used).
template <typename T = bt_list>
std::pair<std::string_view, T> next_list() {
std::pair<std::string_view, T> pair;
pair.first = next_list(pair.second);
return pair;
}
/// Same as above, but takes a pre-existing list-like data type. Returns the key.
template <typename T>
std::string_view next_list(T& list) {
if (!is_list()) throw bt_deserialize_invalid_type{"next bt value is not a list"};
bt_list_consumer::consume_list(list);
return flush_key();
}
/// Consumes a string->dict pair, return it as a dict-like type. This typically requires
/// dynamic allocation, but only has to parse the data once. Compare with consume_dict_data()
/// which allows alloc-free traversal, but requires parsing twice (if the contents are to be
/// used).
template <typename T = bt_dict>
std::pair<std::string_view, T> next_dict() {
std::pair<std::string_view, T> pair;
pair.first = consume_dict(pair.second);
return pair;
}
/// Same as above, but takes a pre-existing dict-like data type. Returns the key.
template <typename T>
std::string_view next_dict(T& dict) {
if (!is_dict()) throw bt_deserialize_invalid_type{"next bt value is not a dict"};
bt_list_consumer::consume_dict(dict);
return flush_key();
}
/// Attempts to parse the next value as a string->list pair and returns the string_view that
/// contains the entire thing. This is recursive into both lists and dicts and likely to be
/// quite inefficient for large, nested structures (unless the values only need to be skipped
/// but aren't separately needed). This, however, does not require dynamic memory allocation.
std::pair<std::string_view, std::string_view> next_list_data() {
if (data.size() < 2 || !is_list()) throw bt_deserialize_invalid_type{"next bt dict value is not a list"};
return {flush_key(), bt_list_consumer::consume_list_data()};
}
/// Same as next_list_data(), but wraps the value in a bt_list_consumer for convenience
std::pair<std::string_view, bt_list_consumer> next_list_consumer() { return next_list_data(); }
/// Attempts to parse the next value as a string->dict pair and returns the string_view that
/// contains the entire thing. This is recursive into both lists and dicts and likely to be
/// quite inefficient for large, nested structures (unless the values only need to be skipped
/// but aren't separately needed). This, however, does not require dynamic memory allocation.
std::pair<std::string_view, std::string_view> next_dict_data() {
if (data.size() < 2 || !is_dict()) throw bt_deserialize_invalid_type{"next bt dict value is not a dict"};
return {flush_key(), bt_list_consumer::consume_dict_data()};
}
/// Same as next_dict_data(), but wraps the value in a bt_dict_consumer for convenience
std::pair<std::string_view, bt_dict_consumer> next_dict_consumer() { return next_dict_data(); }
/// Skips ahead until we find the first key >= the given key or reach the end of the dict.
/// Returns true if we found an exact match, false if we reached some greater value or the end.
/// If we didn't hit the end, the next `consumer_*()` call will return the key-value pair we
/// found (either the exact match or the first key greater than the requested key).
///
/// Two important notes:
///
/// - properly encoded bt dicts must have lexicographically sorted keys, and this method assumes
/// that the input is correctly sorted (and thus if we find a greater value then your key does
/// not exist).
/// - this is irreversible; you cannot returned to skipped values without reparsing. (You *can*
/// however, make a copy of the bt_dict_consumer before calling and use the copy to return to
/// the pre-skipped position).
bool skip_until(std::string_view find) {
while (consume_key() && key_ < find) {
flush_key();
skip_value();
}
return key_ == find;
}
/// The `consume_*` functions are wrappers around next_whatever that discard the returned key.
///
/// Intended for use with skip_until such as:
///
/// std::string value;
/// if (d.skip_until("key"))
/// value = d.consume_string();
///
auto consume_string_view() { return next_string().second; }
auto consume_string() { return std::string{consume_string_view()}; }
template <typename IntType>
auto consume_integer() { return next_integer<IntType>().second; }
template <typename T = bt_list>
auto consume_list() { return next_list<T>().second; }
template <typename T>
void consume_list(T& list) { next_list(list); }
template <typename T = bt_dict>
auto consume_dict() { return next_dict<T>().second; }
template <typename T>
void consume_dict(T& dict) { next_dict(dict); }
std::string_view consume_list_data() { return next_list_data().second; }
std::string_view consume_dict_data() { return next_dict_data().second; }
bt_list_consumer consume_list_consumer() { return consume_list_data(); }
bt_dict_consumer consume_dict_consumer() { return consume_dict_data(); }
};
} // namespace oxenmq

112
oxenmq/bt_value.h Normal file
View File

@ -0,0 +1,112 @@
// Copyright (c) 2019-2021, The Oxen Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
// This header is here to provide just the basic bt_value/bt_dict/bt_list definitions without
// needing to include the full bt_serialize.h header.
#include <map>
#include <list>
#include <cstdint>
#include <variant>
#include <string>
#include <string_view>
namespace oxenmq {
struct bt_value;
/// The type used to store dictionaries inside bt_value.
using bt_dict = std::map<std::string, bt_value>; // NB: unordered_map doesn't work because it can't be used with a predeclared type
/// The type used to store list items inside bt_value.
using bt_list = std::list<bt_value>;
/// The basic variant that can hold anything (recursively).
using bt_variant = std::variant<
std::string,
std::string_view,
int64_t,
uint64_t,
bt_list,
bt_dict
>;
#ifdef __cpp_lib_remove_cvref // C++20
using std::remove_cvref_t;
#else
template <typename T>
using remove_cvref_t = std::remove_cv_t<std::remove_reference_t<T>>;
#endif
template <typename T, typename Variant>
struct has_alternative;
template <typename T, typename... V>
struct has_alternative<T, std::variant<V...>> : std::bool_constant<(std::is_same_v<T, V> || ...)> {};
template <typename T, typename Variant>
constexpr bool has_alternative_v = has_alternative<T, Variant>::value;
namespace detail {
template <typename Tuple, size_t... Is>
bt_list tuple_to_list(const Tuple& tuple, std::index_sequence<Is...>) {
return {{bt_value{std::get<Is>(tuple)}...}};
}
template <typename T> constexpr bool is_tuple = false;
template <typename... T> constexpr bool is_tuple<std::tuple<T...>> = true;
template <typename S, typename T> constexpr bool is_tuple<std::pair<S, T>> = true;
}
/// Recursive generic type that can fully represent everything valid for a BT serialization.
/// This is basically just an empty wrapper around the std::variant, except we add some extra
/// converting constructors:
/// - integer constructors so that any unsigned value goes to the uint64_t and any signed value goes
/// to the int64_t.
/// - std::tuple and std::pair constructors that build a bt_list out of the tuple/pair elements.
struct bt_value : bt_variant {
using bt_variant::bt_variant;
using bt_variant::operator=;
template <typename T, typename U = std::remove_reference_t<T>, std::enable_if_t<std::is_integral_v<U> && std::is_unsigned_v<U>, int> = 0>
bt_value(T&& uint) : bt_variant{static_cast<uint64_t>(uint)} {}
template <typename T, typename U = std::remove_reference_t<T>, std::enable_if_t<std::is_integral_v<U> && std::is_signed_v<U>, int> = 0>
bt_value(T&& sint) : bt_variant{static_cast<int64_t>(sint)} {}
template <typename... T>
bt_value(const std::tuple<T...>& tuple) : bt_variant{detail::tuple_to_list(tuple, std::index_sequence_for<T...>{})} {}
template <typename S, typename T>
bt_value(const std::pair<S, T>& pair) : bt_variant{detail::tuple_to_list(pair, std::index_sequence_for<S, T>{})} {}
template <typename T, typename U = std::remove_reference_t<T>, std::enable_if_t<!std::is_integral_v<U> && !detail::is_tuple<U>, int> = 0>
bt_value(T&& v) : bt_variant{std::forward<T>(v)} {}
bt_value(const char* s) : bt_value{std::string_view{s}} {}
};
}

28
oxenmq/byte_type.h Normal file
View File

@ -0,0 +1,28 @@
#pragma once
// Specializations for assigning from a char into an output iterator, used by hex/base32z/base64
// decoding to bytes.
#include <iterator>
#include <type_traits>
namespace oxenmq::detail {
// Fallback - we just try a char
template <typename OutputIt, typename = void>
struct byte_type { using type = char; };
// Support for things like std::back_inserter:
template <typename OutputIt>
struct byte_type<OutputIt, std::void_t<typename OutputIt::container_type>> {
using type = typename OutputIt::container_type::value_type; };
// iterator, raw pointers:
template <typename OutputIt>
struct byte_type<OutputIt, std::enable_if_t<std::is_reference_v<typename std::iterator_traits<OutputIt>::reference>>> {
using type = std::remove_reference_t<typename std::iterator_traits<OutputIt>::reference>; };
template <typename OutputIt>
using byte_type_t = typename byte_type<OutputIt>::type;
}

36
lokimq/connections.cpp → oxenmq/connections.cpp
View File

@ -1,8 +1,8 @@
#include "lokimq.h"
#include "lokimq-internal.h"
#include "oxenmq.h"
#include "oxenmq-internal.h"
#include "hex.h"
namespace lokimq {
namespace oxenmq {
std::ostream& operator<<(std::ostream& o, const ConnectionID& conn) {
if (!conn.pk.empty())
@ -24,7 +24,7 @@ void add_pollitem(std::vector<zmq::pollitem_t>& pollitems, zmq::socket_t& sock)
} // anonymous namespace
void LokiMQ::rebuild_pollitems() {
void OxenMQ::rebuild_pollitems() {
pollitems.clear();
add_pollitem(pollitems, command);
add_pollitem(pollitems, workers_socket);
@ -35,7 +35,7 @@ void LokiMQ::rebuild_pollitems() {
pollitems_stale = false;
}
void LokiMQ::setup_external_socket(zmq::socket_t& socket) {
void OxenMQ::setup_external_socket(zmq::socket_t& socket) {
socket.set(zmq::sockopt::reconnect_ivl, (int) RECONNECT_INTERVAL.count());
socket.set(zmq::sockopt::reconnect_ivl_max, (int) RECONNECT_INTERVAL_MAX.count());
socket.set(zmq::sockopt::handshake_ivl, (int) HANDSHAKE_TIME.count());
@ -47,7 +47,7 @@ void LokiMQ::setup_external_socket(zmq::socket_t& socket) {
}
}
void LokiMQ::setup_outgoing_socket(zmq::socket_t& socket, std::string_view remote_pubkey) {
void OxenMQ::setup_outgoing_socket(zmq::socket_t& socket, std::string_view remote_pubkey) {
setup_external_socket(socket);
@ -67,7 +67,7 @@ void LokiMQ::setup_outgoing_socket(zmq::socket_t& socket, std::string_view remot
// else let ZMQ pick a random one
}
ConnectionID LokiMQ::connect_sn(std::string_view pubkey, std::chrono::milliseconds keep_alive, std::string_view hint) {
ConnectionID OxenMQ::connect_sn(std::string_view pubkey, std::chrono::milliseconds keep_alive, std::string_view hint) {
if (!proxy_thread.joinable())
throw std::logic_error("Cannot call connect_sn() before calling `start()`");
@ -76,7 +76,7 @@ ConnectionID LokiMQ::connect_sn(std::string_view pubkey, std::chrono::millisecon
return pubkey;
}
ConnectionID LokiMQ::connect_remote(const address& remote, ConnectSuccess on_connect, ConnectFailure on_failure,
ConnectionID OxenMQ::connect_remote(const address& remote, ConnectSuccess on_connect, ConnectFailure on_failure,
AuthLevel auth_level, std::chrono::milliseconds timeout) {
if (!proxy_thread.joinable())
throw std::logic_error("Cannot call connect_remote() before calling `start()`");
@ -96,13 +96,13 @@ ConnectionID LokiMQ::connect_remote(const address& remote, ConnectSuccess on_con
return id;
}
ConnectionID LokiMQ::connect_remote(std::string_view remote, ConnectSuccess on_connect, ConnectFailure on_failure,
ConnectionID OxenMQ::connect_remote(std::string_view remote, ConnectSuccess on_connect, ConnectFailure on_failure,
std::string_view pubkey, AuthLevel auth_level, std::chrono::milliseconds timeout) {
return connect_remote(address{remote}.set_pubkey(pubkey),
std::move(on_connect), std::move(on_failure), auth_level, timeout);
}
void LokiMQ::disconnect(ConnectionID id, std::chrono::milliseconds linger) {
void OxenMQ::disconnect(ConnectionID id, std::chrono::milliseconds linger) {
detail::send_control(get_control_socket(), "DISCONNECT", bt_serialize<bt_dict>({
{"conn_id", id.id},
{"linger_ms", linger.count()},
@ -111,7 +111,7 @@ void LokiMQ::disconnect(ConnectionID id, std::chrono::milliseconds linger) {
}
std::pair<zmq::socket_t *, std::string>
LokiMQ::proxy_connect_sn(std::string_view remote, std::string_view connect_hint, bool optional, bool incoming_only, bool outgoing_only, std::chrono::milliseconds keep_alive) {
OxenMQ::proxy_connect_sn(std::string_view remote, std::string_view connect_hint, bool optional, bool incoming_only, bool outgoing_only, std::chrono::milliseconds keep_alive) {
ConnectionID remote_cid{remote};
auto its = peers.equal_range(remote_cid);
peer_info* peer = nullptr;
@ -186,7 +186,7 @@ LokiMQ::proxy_connect_sn(std::string_view remote, std::string_view connect_hint,
return {&connections.back(), ""s};
}
std::pair<zmq::socket_t *, std::string> LokiMQ::proxy_connect_sn(bt_dict_consumer data) {
std::pair<zmq::socket_t *, std::string> OxenMQ::proxy_connect_sn(bt_dict_consumer data) {
std::string_view hint, remote_pk;
std::chrono::milliseconds keep_alive;
bool optional = false, incoming_only = false, outgoing_only = false;
@ -226,7 +226,7 @@ void update_connection_indices(Container& c, size_t index, AccessIndex get_index
/// Closes outgoing connections and removes all references. Note that this will call `erase()`
/// which can invalidate iterators on the various connection containers - if you don't want that,
/// delete it first so that the container won't contain the element being deleted.
void LokiMQ::proxy_close_connection(size_t index, std::chrono::milliseconds linger) {
void OxenMQ::proxy_close_connection(size_t index, std::chrono::milliseconds linger) {
connections[index].set(zmq::sockopt::linger, linger > 0ms ? (int) linger.count() : 0);
pollitems_stale = true;
connections.erase(connections.begin() + index);
@ -244,7 +244,7 @@ void LokiMQ::proxy_close_connection(size_t index, std::chrono::milliseconds ling
conn_index_to_id.erase(conn_index_to_id.begin() + index);
}
void LokiMQ::proxy_expire_idle_peers() {
void OxenMQ::proxy_expire_idle_peers() {
for (auto it = peers.begin(); it != peers.end(); ) {
auto &info = it->second;
if (info.outgoing()) {
@ -267,7 +267,7 @@ void LokiMQ::proxy_expire_idle_peers() {
}
}
void LokiMQ::proxy_conn_cleanup() {
void OxenMQ::proxy_conn_cleanup() {
LMQ_TRACE("starting proxy connections cleanup");
// Drop idle connections (if we haven't done it in a while)
@ -307,7 +307,7 @@ void LokiMQ::proxy_conn_cleanup() {
LMQ_TRACE("done proxy connections cleanup");
};
void LokiMQ::proxy_connect_remote(bt_dict_consumer data) {
void OxenMQ::proxy_connect_remote(bt_dict_consumer data) {
AuthLevel auth_level = AuthLevel::none;
long long conn_id = -1;
ConnectSuccess on_connect;
@ -372,7 +372,7 @@ void LokiMQ::proxy_connect_remote(bt_dict_consumer data) {
peers.emplace(std::move(conn), std::move(peer));
}
void LokiMQ::proxy_disconnect(bt_dict_consumer data) {
void OxenMQ::proxy_disconnect(bt_dict_consumer data) {
ConnectionID connid{-1};
std::chrono::milliseconds linger = 1s;
@ -388,7 +388,7 @@ void LokiMQ::proxy_disconnect(bt_dict_consumer data) {
proxy_disconnect(std::move(connid), linger);
}
void LokiMQ::proxy_disconnect(ConnectionID conn, std::chrono::milliseconds linger) {
void OxenMQ::proxy_disconnect(ConnectionID conn, std::chrono::milliseconds linger) {
LMQ_TRACE("Disconnecting outgoing connection to ", conn);
auto pr = peers.equal_range(conn);
for (auto it = pr.first; it != pr.second; ++it) {

96
oxenmq/connections.h Normal file
View File

@ -0,0 +1,96 @@
#pragma once
#include "auth.h"
#include "bt_value.h"
#include <string_view>
#include <iosfwd>
#include <stdexcept>
#include <string>
#include <utility>
#include <variant>
namespace oxenmq {
struct ConnectionID;
namespace detail {
template <typename... T>
bt_dict build_send(ConnectionID to, std::string_view cmd, T&&... opts);
}
/// Opaque data structure representing a connection which supports ==, !=, < and std::hash. For
/// connections to service node this is the service node pubkey (and you can pass a 32-byte string
/// anywhere a ConnectionID is called for). For non-SN remote connections you need to keep a copy
/// of the ConnectionID returned by connect_remote().
struct ConnectionID {
// Default construction; creates a ConnectionID with an invalid internal ID that will not match
// an actual connection.
ConnectionID() : ConnectionID(0) {}
// Construction from a service node pubkey
ConnectionID(std::string pubkey_) : id{SN_ID}, pk{std::move(pubkey_)} {
if (pk.size() != 32)
throw std::runtime_error{"Invalid pubkey: expected 32 bytes"};
}
// Construction from a service node pubkey
ConnectionID(std::string_view pubkey_) : ConnectionID(std::string{pubkey_}) {}
ConnectionID(const ConnectionID&) = default;
ConnectionID(ConnectionID&&) = default;
ConnectionID& operator=(const ConnectionID&) = default;
ConnectionID& operator=(ConnectionID&&) = default;
// Returns true if this is a ConnectionID (false for a default-constructed, invalid id)
explicit operator bool() const {
return id != 0;
}
// Two ConnectionIDs are equal if they are both SNs and have matching pubkeys, or they are both
// not SNs and have matching internal IDs and routes. (Pubkeys do not have to match for
// non-SNs).
bool operator==(const ConnectionID &o) const {
if (sn() && o.sn())
return pk == o.pk;
return id == o.id && route == o.route;
}
bool operator!=(const ConnectionID &o) const { return !(*this == o); }
bool operator<(const ConnectionID &o) const {
if (sn() && o.sn())
return pk < o.pk;
return id < o.id || (id == o.id && route < o.route);
}
// Returns true if this ConnectionID represents a SN connection
bool sn() const { return id == SN_ID; }
// Returns this connection's pubkey, if any. (Note that all curve connections have pubkeys, not
// only SNs).
const std::string& pubkey() const { return pk; }
// Returns a copy of the ConnectionID with the route set to empty.
ConnectionID unrouted() { return ConnectionID{id, pk, ""}; }
private:
ConnectionID(long long id) : id{id} {}
ConnectionID(long long id, std::string pubkey, std::string route = "")
: id{id}, pk{std::move(pubkey)}, route{std::move(route)} {}
constexpr static long long SN_ID = -1;
long long id = 0;
std::string pk;
std::string route;
friend class OxenMQ;
friend struct std::hash<ConnectionID>;
template <typename... T>
friend bt_dict detail::build_send(ConnectionID to, std::string_view cmd, T&&... opts);
friend std::ostream& operator<<(std::ostream& o, const ConnectionID& conn);
};
} // namespace oxenmq
namespace std {
template <> struct hash<oxenmq::ConnectionID> {
size_t operator()(const oxenmq::ConnectionID &c) const {
return c.sn() ? oxenmq::already_hashed{}(c.pk) :
std::hash<long long>{}(c.id) + std::hash<std::string>{}(c.route);
}
};
} // namespace std

165
oxenmq/hex.h Normal file
View File

@ -0,0 +1,165 @@
// Copyright (c) 2019-2021, The Oxen Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <string>
#include <string_view>
#include <array>
#include <iterator>
#include <cassert>
#include "byte_type.h"
namespace oxenmq {
namespace detail {
/// Compile-time generated lookup tables hex conversion
struct hex_table {
char from_hex_lut[256];
char to_hex_lut[16];
constexpr hex_table() noexcept : from_hex_lut{}, to_hex_lut{} {
for (unsigned char c = 0; c < 10; c++) {
from_hex_lut[(unsigned char)('0' + c)] = 0 + c;
to_hex_lut[ (unsigned char)( 0 + c)] = '0' + c;
}
for (unsigned char c = 0; c < 6; c++) {
from_hex_lut[(unsigned char)('a' + c)] = 10 + c;
from_hex_lut[(unsigned char)('A' + c)] = 10 + c;
to_hex_lut[ (unsigned char)(10 + c)] = 'a' + c;
}
}
constexpr char from_hex(unsigned char c) const noexcept { return from_hex_lut[c]; }
constexpr char to_hex(unsigned char b) const noexcept { return to_hex_lut[b]; }
} constexpr hex_lut;
// This main point of this static assert is to force the compiler to compile-time build the constexpr tables.
static_assert(hex_lut.from_hex('a') == 10 && hex_lut.from_hex('F') == 15 && hex_lut.to_hex(13) == 'd', "");
} // namespace detail
/// Creates hex digits from a character sequence.
template <typename InputIt, typename OutputIt>
void to_hex(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "to_hex requires chars/bytes");
for (; begin != end; ++begin) {
uint8_t c = static_cast<uint8_t>(*begin);
*out++ = detail::hex_lut.to_hex(c >> 4);
*out++ = detail::hex_lut.to_hex(c & 0x0f);
}
}
/// Creates a string of hex digits from a character sequence iterator pair
template <typename It>
std::string to_hex(It begin, It end) {
std::string hex;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>)
hex.reserve(2 * std::distance(begin, end));
to_hex(begin, end, std::back_inserter(hex));
return hex;
}
/// Creates a hex string from an iterable, std::string-like object
template <typename CharT>
std::string to_hex(std::basic_string_view<CharT> s) { return to_hex(s.begin(), s.end()); }
inline std::string to_hex(std::string_view s) { return to_hex<>(s); }
/// Returns true if the given value is a valid hex digit.
template <typename CharT>
constexpr bool is_hex_digit(CharT c) {
static_assert(sizeof(CharT) == 1, "is_hex requires chars/bytes");
return detail::hex_lut.from_hex(static_cast<unsigned char>(c)) != 0 || static_cast<unsigned char>(c) == '0';
}
/// Returns true if all elements in the range are hex characters *and* the string length is a
/// multiple of 2, and thus suitable to pass to from_hex().
template <typename It>
constexpr bool is_hex(It begin, It end) {
static_assert(sizeof(decltype(*begin)) == 1, "is_hex requires chars/bytes");
constexpr bool ra = std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>;
if constexpr (ra)
if (std::distance(begin, end) % 2 != 0)
return false;
size_t count = 0;
for (; begin != end; ++begin) {
if constexpr (!ra) ++count;
if (!is_hex_digit(*begin))
return false;
}
if constexpr (!ra)
return count % 2 == 0;
return true;
}
/// Returns true if all elements in the string-like value are hex characters
template <typename CharT>
constexpr bool is_hex(std::basic_string_view<CharT> s) { return is_hex(s.begin(), s.end()); }
constexpr bool is_hex(std::string_view s) { return is_hex(s.begin(), s.end()); }
/// Convert a hex digit into its numeric (0-15) value
constexpr char from_hex_digit(unsigned char x) noexcept {
return detail::hex_lut.from_hex(x);
}
/// Constructs a byte value from a pair of hex digits
constexpr char from_hex_pair(unsigned char a, unsigned char b) noexcept { return (from_hex_digit(a) << 4) | from_hex_digit(b); }
/// Converts a sequence of hex digits to bytes. Undefined behaviour if any characters are not in
/// [0-9a-fA-F] or if the input sequence length is not even: call `is_hex` first if you need to
/// check. It is permitted for the input and output ranges to overlap as long as out is no earlier
/// than begin.
template <typename InputIt, typename OutputIt>
void from_hex(InputIt begin, InputIt end, OutputIt out) {
using std::distance;
assert(is_hex(begin, end));
while (begin != end) {
auto a = *begin++;
auto b = *begin++;
*out++ = static_cast<detail::byte_type_t<OutputIt>>(
from_hex_pair(static_cast<unsigned char>(a), static_cast<unsigned char>(b)));
}
}
/// Converts a sequence of hex digits to a string of bytes and returns it. Undefined behaviour if
/// the input sequence is not an even-length sequence of [0-9a-fA-F] characters.
template <typename It>
std::string from_hex(It begin, It end) {
std::string bytes;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>)
bytes.reserve(std::distance(begin, end) / 2);
from_hex(begin, end, std::back_inserter(bytes));
return bytes;
}
/// Converts hex digits from a std::string-like object into a std::string of bytes. Undefined
/// behaviour if any characters are not in [0-9a-fA-F] or if the input sequence length is not even.
template <typename CharT>
std::string from_hex(std::basic_string_view<CharT> s) { return from_hex(s.begin(), s.end()); }
inline std::string from_hex(std::string_view s) { return from_hex<>(s); }
}

32
lokimq/jobs.cpp → oxenmq/jobs.cpp
View File

@ -1,10 +1,10 @@
#include "lokimq.h"
#include "oxenmq.h"
#include "batch.h"
#include "lokimq-internal.h"
#include "oxenmq-internal.h"
namespace lokimq {
namespace oxenmq {
void LokiMQ::proxy_batch(detail::Batch* batch) {
void OxenMQ::proxy_batch(detail::Batch* batch) {
batches.insert(batch);
const auto [jobs, tagged_threads] = batch->size();
LMQ_TRACE("proxy queuing batch job with ", jobs, " jobs", tagged_threads ? " (job uses tagged thread(s))" : "");
@ -26,7 +26,7 @@ void LokiMQ::proxy_batch(detail::Batch* batch) {
proxy_skip_one_poll = true;
}
void LokiMQ::job(std::function<void()> f, std::optional<TaggedThreadID> thread) {
void OxenMQ::job(std::function<void()> f, std::optional<TaggedThreadID> thread) {
if (thread && thread->_id == -1)
throw std::logic_error{"job() cannot be used to queue an in-proxy job"};
auto* b = new Batch<void>;
@ -35,7 +35,7 @@ void LokiMQ::job(std::function<void()> f, std::optional<TaggedThreadID> thread)
detail::send_control(get_control_socket(), "BATCH", bt_serialize(reinterpret_cast<uintptr_t>(baseptr)));
}
void LokiMQ::proxy_schedule_reply_job(std::function<void()> f) {
void OxenMQ::proxy_schedule_reply_job(std::function<void()> f) {
auto* b = new Batch<void>;
b->add_job(std::move(f));
batches.insert(b);
@ -43,7 +43,7 @@ void LokiMQ::proxy_schedule_reply_job(std::function<void()> f) {
proxy_skip_one_poll = true;
}
void LokiMQ::proxy_run_batch_jobs(std::queue<batch_job>& jobs, const int reserved, int& active, bool reply) {
void OxenMQ::proxy_run_batch_jobs(std::queue<batch_job>& jobs, const int reserved, int& active, bool reply) {
while (!jobs.empty() && active_workers() < max_workers &&
(active < reserved || active_workers() < general_workers)) {
proxy_run_worker(get_idle_worker().load(std::move(jobs.front()), reply));
@ -54,20 +54,20 @@ void LokiMQ::proxy_run_batch_jobs(std::queue<batch_job>& jobs, const int reserve
// Called either within the proxy thread, or before the proxy thread has been created; actually adds
// the timer. If the timer object hasn't been set up yet it gets set up here.
void LokiMQ::proxy_timer(std::function<void()> job, std::chrono::milliseconds interval, bool squelch, int thread) {
void OxenMQ::proxy_timer(std::function<void()> job, std::chrono::milliseconds interval, bool squelch, int thread) {
if (!timers)
timers.reset(zmq_timers_new());
int timer_id = zmq_timers_add(timers.get(),
interval.count(),
[](int timer_id, void* self) { static_cast<LokiMQ*>(self)->_queue_timer_job(timer_id); },
[](int timer_id, void* self) { static_cast<OxenMQ*>(self)->_queue_timer_job(timer_id); },
this);
if (timer_id == -1)
throw zmq::error_t{};
timer_jobs[timer_id] = { std::move(job), squelch, false, thread };
}
void LokiMQ::proxy_timer(bt_list_consumer timer_data) {
void OxenMQ::proxy_timer(bt_list_consumer timer_data) {
std::unique_ptr<std::function<void()>> func{reinterpret_cast<std::function<void()>*>(timer_data.consume_integer<uintptr_t>())};
auto interval = std::chrono::milliseconds{timer_data.consume_integer<uint64_t>()};
auto squelch = timer_data.consume_integer<bool>();
@ -77,7 +77,7 @@ void LokiMQ::proxy_timer(bt_list_consumer timer_data) {
proxy_timer(std::move(*func), interval, squelch, thread);
}
void LokiMQ::_queue_timer_job(int timer_id) {
void OxenMQ::_queue_timer_job(int timer_id) {
auto it = timer_jobs.find(timer_id);
if (it == timer_jobs.end()) {
LMQ_LOG(warn, "Could not find timer job ", timer_id);
@ -107,7 +107,7 @@ void LokiMQ::_queue_timer_job(int timer_id) {
auto it = timer_jobs.find(timer_id);
if (it != timer_jobs.end())
it->second.running = false;
}, LokiMQ::run_in_proxy);
}, OxenMQ::run_in_proxy);
}
batches.insert(b);
LMQ_TRACE("b: ", b->size().first, ", ", b->size().second, "; thread = ", thread);
@ -118,7 +118,7 @@ void LokiMQ::_queue_timer_job(int timer_id) {
queue.emplace(static_cast<detail::Batch*>(b), 0);
}
void LokiMQ::add_timer(std::function<void()> job, std::chrono::milliseconds interval, bool squelch, std::optional<TaggedThreadID> thread) {
void OxenMQ::add_timer(std::function<void()> job, std::chrono::milliseconds interval, bool squelch, std::optional<TaggedThreadID> thread) {
int th_id = thread ? thread->_id : 0;
if (proxy_thread.joinable()) {
detail::send_control(get_control_socket(), "TIMER", bt_serialize(bt_list{{
@ -131,9 +131,9 @@ void LokiMQ::add_timer(std::function<void()> job, std::chrono::milliseconds inte
}
}
void LokiMQ::TimersDeleter::operator()(void* timers) { zmq_timers_destroy(&timers); }
void OxenMQ::TimersDeleter::operator()(void* timers) { zmq_timers_destroy(&timers); }
TaggedThreadID LokiMQ::add_tagged_thread(std::string name, std::function<void()> start) {
TaggedThreadID OxenMQ::add_tagged_thread(std::string name, std::function<void()> start) {
if (proxy_thread.joinable())
throw std::logic_error{"Cannot add tagged threads after calling `start()`"};
@ -146,7 +146,7 @@ TaggedThreadID LokiMQ::add_tagged_thread(std::string name, std::function<void()>
run.worker_routing_id = "t" + std::to_string(run.worker_id);
LMQ_TRACE("Created new tagged thread ", name, " with routing id ", run.worker_routing_id);
run.worker_thread = std::thread{&LokiMQ::worker_thread, this, run.worker_id, name, std::move(start)};
run.worker_thread = std::thread{&OxenMQ::worker_thread, this, run.worker_id, name, std::move(start)};
return TaggedThreadID{static_cast<int>(run.worker_id)};
}

57
oxenmq/message.h Normal file
View File

@ -0,0 +1,57 @@
#pragma once
#include <vector>
#include "connections.h"
namespace oxenmq {
class OxenMQ;
/// Encapsulates an incoming message from a remote connection with message details plus extra
/// info need to send a reply back through the proxy thread via the `reply()` method. Note that
/// this object gets reused: callbacks should use but not store any reference beyond the callback.
class Message {
public:
OxenMQ& oxenmq; ///< The owning OxenMQ object
std::vector<std::string_view> data; ///< The provided command data parts, if any.
ConnectionID conn; ///< The connection info for routing a reply; also contains the pubkey/sn status.
std::string reply_tag; ///< If the invoked command is a request command this is the required reply tag that will be prepended by `send_reply()`.
Access access; ///< The access level of the invoker. This can be higher than the access level of the command, for example for an admin invoking a basic command.
std::string remote; ///< Some sort of remote address from which the request came. Often "IP" for TCP connections and "localhost:UID:GID:PID" for UDP connections.
/// Constructor
Message(OxenMQ& lmq, ConnectionID cid, Access access, std::string remote)
: oxenmq{lmq}, conn{std::move(cid)}, access{std::move(access)}, remote{std::move(remote)} {}
// Non-copyable
Message(const Message&) = delete;
Message& operator=(const Message&) = delete;
/// Sends a command back to whomever sent this message. Arguments are forwarded to send() but
/// with send_option::optional{} added if the originator is not a SN. For SN messages (i.e.
/// where `sn` is true) this is a "strong" reply by default in that the proxy will attempt to
/// establish a new connection to the SN if no longer connected. For non-SN messages the reply
/// will be attempted using the available routing information, but if the connection has already
/// been closed the reply will be dropped.
///
/// If you want to send a non-strong reply even when the remote is a service node then add
/// an explicit `send_option::optional()` argument.
template <typename... Args>
void send_back(std::string_view, Args&&... args);
/// Sends a reply to a request. This takes no command: the command is always the built-in
/// "REPLY" command, followed by the unique reply tag, then any reply data parts. All other
/// arguments are as in `send_back()`. You should only send one reply for a command expecting
/// replies, though this is not enforced: attempting to send multiple replies will simply be
/// dropped when received by the remote. (Note, however, that it is possible to send multiple
/// messages -- e.g. you could send a reply and then also call send_back() and/or send_request()
/// to send more requests back to the sender).
template <typename... Args>
void send_reply(Args&&... args);
/// Sends a request back to whomever sent this message. This is effectively a wrapper around
/// lmq.request() that takes care of setting up the recipient arguments.
template <typename ReplyCallback, typename... Args>
void send_request(std::string_view cmd, ReplyCallback&& callback, Args&&... args);
};
}

4
lokimq/lokimq-internal.h → oxenmq/oxenmq-internal.h
View File

@ -1,5 +1,5 @@
#pragma once
#include "lokimq.h"
#include "oxenmq.h"
// Inside some method:
// LMQ_LOG(warn, "bad ", 42, " stuff");
@ -13,7 +13,7 @@
# define LMQ_TRACE(...)
#endif
namespace lokimq {
namespace oxenmq {
constexpr char SN_ADDR_COMMAND[] = "inproc://sn-command";
constexpr char SN_ADDR_WORKERS[] = "inproc://sn-workers";

76
lokimq/lokimq.cpp → oxenmq/oxenmq.cpp
View File

@ -1,5 +1,5 @@
#include "lokimq.h"
#include "lokimq-internal.h"
#include "oxenmq.h"
#include "oxenmq-internal.h"
#include "zmq.hpp"
#include <map>
#include <random>
@ -13,7 +13,7 @@ extern "C" {
}
#include "hex.h"
namespace lokimq {
namespace oxenmq {
namespace {
@ -76,20 +76,20 @@ std::pair<std::string, AuthLevel> extract_metadata(zmq::message_t& msg) {
} // namespace detail
void LokiMQ::set_zmq_context_option(zmq::ctxopt option, int value) {
void OxenMQ::set_zmq_context_option(zmq::ctxopt option, int value) {
context.set(option, value);
}
void LokiMQ::log_level(LogLevel level) {
void OxenMQ::log_level(LogLevel level) {
log_lvl.store(level, std::memory_order_relaxed);
}
LogLevel LokiMQ::log_level() const {
LogLevel OxenMQ::log_level() const {
return log_lvl.load(std::memory_order_relaxed);
}
CatHelper LokiMQ::add_category(std::string name, Access access_level, unsigned int reserved_threads, int max_queue) {
CatHelper OxenMQ::add_category(std::string name, Access access_level, unsigned int reserved_threads, int max_queue) {
check_not_started(proxy_thread, "add a category");
if (name.size() > MAX_CATEGORY_LENGTH)
@ -107,7 +107,7 @@ CatHelper LokiMQ::add_category(std::string name, Access access_level, unsigned i
return ret;
}
void LokiMQ::add_command(const std::string& category, std::string name, CommandCallback callback) {
void OxenMQ::add_command(const std::string& category, std::string name, CommandCallback callback) {
check_not_started(proxy_thread, "add a command");
if (name.size() > MAX_COMMAND_LENGTH)
@ -126,12 +126,12 @@ void LokiMQ::add_command(const std::string& category, std::string name, CommandC
throw std::runtime_error("Cannot add command `" + fullname + "': that command already exists");
}
void LokiMQ::add_request_command(const std::string& category, std::string name, CommandCallback callback) {
void OxenMQ::add_request_command(const std::string& category, std::string name, CommandCallback callback) {
add_command(category, name, std::move(callback));
categories.at(category).commands.at(name).second = true;
}
void LokiMQ::add_command_alias(std::string from, std::string to) {
void OxenMQ::add_command_alias(std::string from, std::string to) {
check_not_started(proxy_thread, "add a command alias");
if (from.empty())
@ -160,10 +160,10 @@ std::atomic<int> next_id{1};
/// Accesses a thread-local command socket connected to the proxy's command socket used to issue
/// commands in a thread-safe manner. A mutex is only required here the first time a thread
/// accesses the control socket.
zmq::socket_t& LokiMQ::get_control_socket() {
zmq::socket_t& OxenMQ::get_control_socket() {
assert(proxy_thread.joinable());
// Optimize by caching the last value; LokiMQ is often a singleton and in that case we're
// Optimize by caching the last value; OxenMQ is often a singleton and in that case we're
// going to *always* hit this optimization. Even if it isn't, we're probably likely to need the
// same control socket from the same thread multiple times sequentially so this may still help.
static thread_local int last_id = -1;
@ -174,7 +174,7 @@ zmq::socket_t& LokiMQ::get_control_socket() {
std::lock_guard lock{control_sockets_mutex};
if (proxy_shutting_down)
throw std::runtime_error("Unable to obtain LokiMQ control socket: proxy thread is shutting down");
throw std::runtime_error("Unable to obtain OxenMQ control socket: proxy thread is shutting down");
auto& socket = control_sockets[std::this_thread::get_id()];
if (!socket) {
@ -188,7 +188,7 @@ zmq::socket_t& LokiMQ::get_control_socket() {
}
LokiMQ::LokiMQ(
OxenMQ::OxenMQ(
std::string pubkey_,
std::string privkey_,
bool service_node,
@ -199,17 +199,17 @@ LokiMQ::LokiMQ(
sn_lookup{std::move(lookup)}, log_lvl{level}, logger{std::move(logger)}
{
LMQ_TRACE("Constructing LokiMQ, id=", object_id, ", this=", this);
LMQ_TRACE("Constructing OxenMQ, id=", object_id, ", this=", this);
if (sodium_init() == -1)
throw std::runtime_error{"libsodium initialization failed"};
if (pubkey.empty() != privkey.empty()) {
throw std::invalid_argument("LokiMQ construction failed: one (and only one) of pubkey/privkey is empty. Both must be specified, or both empty to generate a key.");
throw std::invalid_argument("OxenMQ construction failed: one (and only one) of pubkey/privkey is empty. Both must be specified, or both empty to generate a key.");
} else if (pubkey.empty()) {
if (service_node)
throw std::invalid_argument("Cannot construct a service node mode LokiMQ without a keypair");
LMQ_LOG(debug, "generating x25519 keypair for remote-only LokiMQ instance");
throw std::invalid_argument("Cannot construct a service node mode OxenMQ without a keypair");
LMQ_LOG(debug, "generating x25519 keypair for remote-only OxenMQ instance");
pubkey.resize(crypto_box_PUBLICKEYBYTES);
privkey.resize(crypto_box_SECRETKEYBYTES);
crypto_box_keypair(reinterpret_cast<unsigned char*>(&pubkey[0]), reinterpret_cast<unsigned char*>(&privkey[0]));
@ -224,11 +224,11 @@ LokiMQ::LokiMQ(
std::string verify_pubkey(crypto_box_PUBLICKEYBYTES, 0);
crypto_scalarmult_base(reinterpret_cast<unsigned char*>(&verify_pubkey[0]), reinterpret_cast<unsigned char*>(&privkey[0]));
if (verify_pubkey != pubkey)
throw std::invalid_argument("Invalid pubkey/privkey values given to LokiMQ construction: pubkey verification failed");
throw std::invalid_argument("Invalid pubkey/privkey values given to OxenMQ construction: pubkey verification failed");
}
}
void LokiMQ::start() {
void OxenMQ::start() {
if (proxy_thread.joinable())
throw std::logic_error("Cannot call start() multiple times!");
@ -238,19 +238,19 @@ void LokiMQ::start() {
if (bind.empty() && local_service_node)
throw std::invalid_argument{"Cannot create a service node listener with no address(es) to bind"};
LMQ_LOG(info, "Initializing LokiMQ ", bind.empty() ? "remote-only" : "listener", " with pubkey ", to_hex(pubkey));
LMQ_LOG(info, "Initializing OxenMQ ", bind.empty() ? "remote-only" : "listener", " with pubkey ", to_hex(pubkey));
int zmq_socket_limit = context.get(zmq::ctxopt::socket_limit);
if (MAX_SOCKETS > 1 && MAX_SOCKETS <= zmq_socket_limit)
context.set(zmq::ctxopt::max_sockets, MAX_SOCKETS);
else
LMQ_LOG(error, "Not applying LokiMQ::MAX_SOCKETS setting: ", MAX_SOCKETS, " must be in [1, ", zmq_socket_limit, "]");
LMQ_LOG(error, "Not applying OxenMQ::MAX_SOCKETS setting: ", MAX_SOCKETS, " must be in [1, ", zmq_socket_limit, "]");
// We bind `command` here so that the `get_control_socket()` below is always connecting to a
// bound socket, but we do nothing else here: the proxy thread is responsible for everything
// except binding it.
command.bind(SN_ADDR_COMMAND);
proxy_thread = std::thread{&LokiMQ::proxy_loop, this};
proxy_thread = std::thread{&OxenMQ::proxy_loop, this};
LMQ_LOG(debug, "Waiting for proxy thread to get ready...");
auto &control = get_control_socket();
@ -260,14 +260,14 @@ void LokiMQ::start() {
zmq::message_t ready_msg;
std::vector<zmq::message_t> parts;
try { recv_message_parts(control, parts); }
catch (const zmq::error_t &e) { throw std::runtime_error("Failure reading from LokiMQ::Proxy thread: "s + e.what()); }
catch (const zmq::error_t &e) { throw std::runtime_error("Failure reading from OxenMQ::Proxy thread: "s + e.what()); }
if (!(parts.size() == 1 && view(parts.front()) == "READY"))
throw std::runtime_error("Invalid startup message from proxy thread (didn't get expected READY message)");
LMQ_LOG(debug, "Proxy thread is ready");
}
void LokiMQ::listen_curve(std::string bind_addr, AllowFunc allow_connection) {
void OxenMQ::listen_curve(std::string bind_addr, AllowFunc allow_connection) {
// TODO: there's no particular reason we can't start listening after starting up; just needs to
// be implemented. (But if we can start we'll probably also want to be able to stop, so it's
// more than just binding that needs implementing).
@ -276,7 +276,7 @@ void LokiMQ::listen_curve(std::string bind_addr, AllowFunc allow_connection) {
bind.emplace_back(std::move(bind_addr), bind_data{true, std::move(allow_connection)});
}
void LokiMQ::listen_plain(std::string bind_addr, AllowFunc allow_connection) {
void OxenMQ::listen_plain(std::string bind_addr, AllowFunc allow_connection) {
// TODO: As above.
check_not_started(proxy_thread, "start listening");
@ -284,7 +284,7 @@ void LokiMQ::listen_plain(std::string bind_addr, AllowFunc allow_connection) {
}
std::pair<LokiMQ::category*, const std::pair<LokiMQ::CommandCallback, bool>*> LokiMQ::get_command(std::string& command) {
std::pair<OxenMQ::category*, const std::pair<OxenMQ::CommandCallback, bool>*> OxenMQ::get_command(std::string& command) {
if (command.size() > MAX_CATEGORY_LENGTH + 1 + MAX_COMMAND_LENGTH) {
LMQ_LOG(warn, "Invalid command '", command, "': command too long");
return {};
@ -320,7 +320,7 @@ std::pair<LokiMQ::category*, const std::pair<LokiMQ::CommandCallback, bool>*> Lo
return {&catit->second, &callback_it->second};
}
void LokiMQ::set_batch_threads(int threads) {
void OxenMQ::set_batch_threads(int threads) {
if (proxy_thread.joinable())
throw std::logic_error("Cannot change reserved batch threads after calling `start()`");
if (threads < -1) // -1 is the default which is based on general threads
@ -328,7 +328,7 @@ void LokiMQ::set_batch_threads(int threads) {
batch_jobs_reserved = threads;
}
void LokiMQ::set_reply_threads(int threads) {
void OxenMQ::set_reply_threads(int threads) {
if (proxy_thread.joinable())
throw std::logic_error("Cannot change reserved reply threads after calling `start()`");
if (threads < -1) // -1 is the default which is based on general threads
@ -336,7 +336,7 @@ void LokiMQ::set_reply_threads(int threads) {
reply_jobs_reserved = threads;
}
void LokiMQ::set_general_threads(int threads) {
void OxenMQ::set_general_threads(int threads) {
if (proxy_thread.joinable())
throw std::logic_error("Cannot change general thread count after calling `start()`");
if (threads < 1)
@ -344,7 +344,7 @@ void LokiMQ::set_general_threads(int threads) {
general_workers = threads;
}
LokiMQ::run_info& LokiMQ::run_info::load(category* cat_, std::string command_, ConnectionID conn_, Access access_, std::string remote_,
OxenMQ::run_info& OxenMQ::run_info::load(category* cat_, std::string command_, ConnectionID conn_, Access access_, std::string remote_,
std::vector<zmq::message_t> data_parts_, const std::pair<CommandCallback, bool>* callback_) {
reset();
cat = cat_;
@ -357,7 +357,7 @@ LokiMQ::run_info& LokiMQ::run_info::load(category* cat_, std::string command_, C
return *this;
}
LokiMQ::run_info& LokiMQ::run_info::load(category* cat_, std::string command_, std::string remote_, std::function<void()> callback) {
OxenMQ::run_info& OxenMQ::run_info::load(category* cat_, std::string command_, std::string remote_, std::function<void()> callback) {
reset();
is_injected = true;
cat = cat_;
@ -369,7 +369,7 @@ LokiMQ::run_info& LokiMQ::run_info::load(category* cat_, std::string command_, s
return *this;
}
LokiMQ::run_info& LokiMQ::run_info::load(pending_command&& pending) {
OxenMQ::run_info& OxenMQ::run_info::load(pending_command&& pending) {
if (auto *f = std::get_if<std::function<void()>>(&pending.callback))
return load(&pending.cat, std::move(pending.command), std::move(pending.remote), std::move(*f));
@ -378,7 +378,7 @@ LokiMQ::run_info& LokiMQ::run_info::load(pending_command&& pending) {
std::move(pending.remote), std::move(pending.data_parts), var::get<0>(pending.callback));
}
LokiMQ::run_info& LokiMQ::run_info::load(batch_job&& bj, bool reply_job, int tagged_thread) {
OxenMQ::run_info& OxenMQ::run_info::load(batch_job&& bj, bool reply_job, int tagged_thread) {
reset();
is_batch_job = true;
is_reply_job = reply_job;
@ -389,7 +389,7 @@ LokiMQ::run_info& LokiMQ::run_info::load(batch_job&& bj, bool reply_job, int tag
}
LokiMQ::~LokiMQ() {
OxenMQ::~OxenMQ() {
if (!proxy_thread.joinable()) {
if (!tagged_workers.empty()) {
// This is a bit icky: we have tagged workers that are waiting for a signal on
@ -416,10 +416,10 @@ LokiMQ::~LokiMQ() {
return;
}
LMQ_LOG(info, "LokiMQ shutting down proxy thread");
LMQ_LOG(info, "OxenMQ shutting down proxy thread");
detail::send_control(get_control_socket(), "QUIT");
proxy_thread.join();
LMQ_LOG(info, "LokiMQ proxy thread has stopped");
LMQ_LOG(info, "OxenMQ proxy thread has stopped");
}
std::ostream &operator<<(std::ostream &os, LogLevel lvl) {
@ -443,5 +443,5 @@ std::string make_random_string(size_t size) {
return rando;
}
} // namespace lokimq
} // namespace oxenmq
// vim:sw=4:et

1528
oxenmq/oxenmq.h Normal file
View File

File diff suppressed because it is too large Load Diff

30
lokimq/proxy.cpp → oxenmq/proxy.cpp
View File

@ -1,5 +1,5 @@
#include "lokimq.h"
#include "lokimq-internal.h"
#include "oxenmq.h"
#include "oxenmq-internal.h"
#include "hex.h"
#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
@ -17,9 +17,9 @@ extern "C" {
}
#endif
namespace lokimq {
namespace oxenmq {
void LokiMQ::proxy_quit() {
void OxenMQ::proxy_quit() {
LMQ_LOG(debug, "Received quit command, shutting down proxy thread");
assert(std::none_of(workers.begin(), workers.end(), [](auto& worker) { return worker.worker_thread.joinable(); }));
@ -41,7 +41,7 @@ void LokiMQ::proxy_quit() {
LMQ_LOG(debug, "Proxy thread teardown complete");
}
void LokiMQ::proxy_send(bt_dict_consumer data) {
void OxenMQ::proxy_send(bt_dict_consumer data) {
// NB: bt_dict_consumer goes in alphabetical order
std::string_view hint;
std::chrono::milliseconds keep_alive{DEFAULT_SEND_KEEP_ALIVE};
@ -205,7 +205,7 @@ void LokiMQ::proxy_send(bt_dict_consumer data) {
}
}
void LokiMQ::proxy_reply(bt_dict_consumer data) {
void OxenMQ::proxy_reply(bt_dict_consumer data) {
bool have_conn_id = false;
ConnectionID conn_id{0};
if (data.skip_until("conn_id")) {
@ -250,11 +250,11 @@ void LokiMQ::proxy_reply(bt_dict_consumer data) {
}
}
void LokiMQ::proxy_control_message(std::vector<zmq::message_t>& parts) {
void OxenMQ::proxy_control_message(std::vector<zmq::message_t>& parts) {
// We throw an uncaught exception here because we only generate control messages internally in
// lokimq code: if one of these condition fail it's a lokimq bug.
// oxenmq code: if one of these condition fail it's a oxenmq bug.
if (parts.size() < 2)
throw std::logic_error("LokiMQ bug: Expected 2-3 message parts for a proxy control message");
throw std::logic_error("OxenMQ bug: Expected 2-3 message parts for a proxy control message");
auto route = view(parts[0]), cmd = view(parts[1]);
LMQ_TRACE("control message: ", cmd);
if (parts.size() == 3) {
@ -306,11 +306,11 @@ void LokiMQ::proxy_control_message(std::vector<zmq::message_t>& parts) {
return;
}
}
throw std::runtime_error("LokiMQ bug: Proxy received invalid control command: " +
throw std::runtime_error("OxenMQ bug: Proxy received invalid control command: " +
std::string{cmd} + " (" + std::to_string(parts.size()) + ")");
}
void LokiMQ::proxy_loop() {
void OxenMQ::proxy_loop() {
#if defined(__linux__) || defined(__sun) || defined(__MINGW32__)
pthread_setname_np(pthread_self(), "lmq-proxy");
@ -371,7 +371,7 @@ void LokiMQ::proxy_loop() {
listener.set(zmq::sockopt::router_mandatory, true);
listener.bind(bind[i].first);
LMQ_LOG(info, "LokiMQ listening on ", bind[i].first);
LMQ_LOG(info, "OxenMQ listening on ", bind[i].first);
connections.push_back(std::move(listener));
auto conn_id = next_conn_id++;
@ -547,7 +547,7 @@ static bool is_error_response(std::string_view cmd) {
// Return true if we recognized/handled the builtin command (even if we reject it for whatever
// reason)
bool LokiMQ::proxy_handle_builtin(size_t conn_index, std::vector<zmq::message_t>& parts) {
bool OxenMQ::proxy_handle_builtin(size_t conn_index, std::vector<zmq::message_t>& parts) {
// Doubling as a bool and an offset:
size_t incoming = connections[conn_index].get(zmq::sockopt::type) == ZMQ_ROUTER;
@ -644,7 +644,7 @@ bool LokiMQ::proxy_handle_builtin(size_t conn_index, std::vector<zmq::message_t>
// pre-1.1.0 sent just a plain UNKNOWNCOMMAND (without the actual command); this was not
// useful, but also this response is *expected* for things 1.0.5 didn't understand, like
// FORBIDDEN_SN: so log it only at debug level and move on.
LMQ_LOG(debug, "Received plain UNKNOWNCOMMAND; remote is probably an older lokimq. Ignoring.");
LMQ_LOG(debug, "Received plain UNKNOWNCOMMAND; remote is probably an older oxenmq. Ignoring.");
return true;
}
@ -669,7 +669,7 @@ bool LokiMQ::proxy_handle_builtin(size_t conn_index, std::vector<zmq::message_t>
return false;
}
void LokiMQ::proxy_process_queue() {
void OxenMQ::proxy_process_queue() {
if (max_workers == 0) // shutting down
return;

103
oxenmq/variant.h Normal file
View File

@ -0,0 +1,103 @@
#pragma once
// Workarounds for macos compatibility. On macOS we aren't allowed to touch anything in
// std::variant that could throw if compiling with a target <10.14 because Apple fails hard at
// properly updating their STL. Thus, if compiling in such a mode, we have to introduce
// workarounds.
//
// This header defines a `var` namespace with `var::get` and `var::visit` implementations. On
// everything except broken backwards macos, this is just an alias to `std`. On broken backwards
// macos, we provide implementations that throw std::runtime_error in failure cases since the
// std::bad_variant_access exception can't be touched.
//
// You also get a BROKEN_APPLE_VARIANT macro defined if targetting a problematic mac architecture.
#include <variant>
#ifdef __APPLE__
# include <AvailabilityVersions.h>
# if defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_14
# define BROKEN_APPLE_VARIANT
# endif
#endif
#ifndef BROKEN_APPLE_VARIANT
namespace var = std; // Oh look, actual C++17 support
#else
// Oh look, apple.
namespace var {
// Apple won't let us use std::visit or std::get if targetting some version of macos earlier than
// 10.14 because Apple is awful about not updating their STL. So we have to provide our own, and
// then call these without `std::` -- on crappy macos we'll come here, on everything else we'll ADL
// to the std:: implementation.
template <typename T, typename... Types>
constexpr T& get(std::variant<Types...>& var) {
if (auto* v = std::get_if<T>(&var)) return *v;
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <typename T, typename... Types>
constexpr const T& get(const std::variant<Types...>& var) {
if (auto* v = std::get_if<T>(&var)) return *v;
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <typename T, typename... Types>
constexpr const T&& get(const std::variant<Types...>&& var) {
if (auto* v = std::get_if<T>(&var)) return std::move(*v);
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <typename T, typename... Types>
constexpr T&& get(std::variant<Types...>&& var) {
if (auto* v = std::get_if<T>(&var)) return std::move(*v);
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, typename... Types>
constexpr auto& get(std::variant<Types...>& var) {
if (auto* v = std::get_if<I>(&var)) return *v;
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, typename... Types>
constexpr const auto& get(const std::variant<Types...>& var) {
if (auto* v = std::get_if<I>(&var)) return *v;
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, typename... Types>
constexpr const auto&& get(const std::variant<Types...>&& var) {
if (auto* v = std::get_if<I>(&var)) return std::move(*v);
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, typename... Types>
constexpr auto&& get(std::variant<Types...>&& var) {
if (auto* v = std::get_if<I>(&var)) return std::move(*v);
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, size_t... More, class Visitor, class Variant>
constexpr auto visit_helper(Visitor&& vis, Variant&& var) {
if (var.index() == I)
return vis(var::get<I>(std::forward<Variant>(var)));
else if constexpr (sizeof...(More) > 0)
return visit_helper<More...>(std::forward<Visitor>(vis), std::forward<Variant>(var));
else
throw std::runtime_error{"Bad visit -- variant is valueless"};
}
template <size_t... Is, class Visitor, class Variant>
constexpr auto visit_helper(Visitor&& vis, Variant&& var, std::index_sequence<Is...>) {
return visit_helper<Is...>(std::forward<Visitor>(vis), std::forward<Variant>(var));
}
// Only handle a single variant here because multi-variant invocation is notably harder (and we
// don't need it).
template <class Visitor, class Variant>
constexpr auto visit(Visitor&& vis, Variant&& var) {
return visit_helper(std::forward<Visitor>(vis), std::forward<Variant>(var),
std::make_index_sequence<std::variant_size_v<std::remove_reference_t<Variant>>>{});
}
} // namespace var
#endif

5
oxenmq/version.h.in Normal file
View File

@ -0,0 +1,5 @@
namespace oxenmq {
constexpr int VERSION_MAJOR = @OXENMQ_VERSION_MAJOR@;
constexpr int VERSION_MINOR = @OXENMQ_VERSION_MINOR@;
constexpr int VERSION_PATCH = @OXENMQ_VERSION_PATCH@;
}

26
lokimq/worker.cpp → oxenmq/worker.cpp
View File

@ -1,6 +1,6 @@
#include "lokimq.h"
#include "oxenmq.h"
#include "batch.h"
#include "lokimq-internal.h"
#include "oxenmq-internal.h"
#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
extern "C" {
@ -9,7 +9,7 @@ extern "C" {
}
#endif
namespace lokimq {
namespace oxenmq {
namespace {
@ -17,7 +17,7 @@ namespace {
// received. If "QUIT" was received, replies with "QUITTING" on the socket and closes it, then
// returns false.
[[gnu::always_inline]] inline
bool worker_wait_for(LokiMQ& lmq, zmq::socket_t& sock, std::vector<zmq::message_t>& parts, const std::string_view worker_id, const std::string_view expect) {
bool worker_wait_for(OxenMQ& lmq, zmq::socket_t& sock, std::vector<zmq::message_t>& parts, const std::string_view worker_id, const std::string_view expect) {
while (true) {
lmq.log(LogLevel::debug, __FILE__, __LINE__, "worker ", worker_id, " waiting for ", expect);
parts.clear();
@ -46,7 +46,7 @@ bool worker_wait_for(LokiMQ& lmq, zmq::socket_t& sock, std::vector<zmq::message_
}
void LokiMQ::worker_thread(unsigned int index, std::optional<std::string> tagged, std::function<void()> start) {
void OxenMQ::worker_thread(unsigned int index, std::optional<std::string> tagged, std::function<void()> start) {
std::string routing_id = (tagged ? "t" : "w") + std::to_string(index); // for routing
std::string_view worker_id{tagged ? *tagged : routing_id}; // for debug
@ -72,8 +72,8 @@ void LokiMQ::worker_thread(unsigned int index, std::optional<std::string> tagged
bool waiting_for_command;
if (tagged) {
// If we're a tagged worker then we got started up before LokiMQ started, so we need to wait
// for an all-clear signal from LokiMQ first, then we fire our `start` callback, then we can
// If we're a tagged worker then we got started up before OxenMQ started, so we need to wait
// for an all-clear signal from OxenMQ first, then we fire our `start` callback, then we can
// start waiting for commands in the main loop further down. (We also can't get the
// reference to our `tagged_workers` element until the main proxy threads is running).
@ -159,7 +159,7 @@ void LokiMQ::worker_thread(unsigned int index, std::optional<std::string> tagged
}
LokiMQ::run_info& LokiMQ::get_idle_worker() {
OxenMQ::run_info& OxenMQ::get_idle_worker() {
if (idle_workers.empty()) {
size_t id = workers.size();
assert(workers.capacity() > id);
@ -174,7 +174,7 @@ LokiMQ::run_info& LokiMQ::get_idle_worker() {
return workers[id];
}
void LokiMQ::proxy_worker_message(std::vector<zmq::message_t>& parts) {
void OxenMQ::proxy_worker_message(std::vector<zmq::message_t>& parts) {
// Process messages sent by workers
if (parts.size() != 2) {
LMQ_LOG(error, "Received send invalid ", parts.size(), "-part message");
@ -268,14 +268,14 @@ void LokiMQ::proxy_worker_message(std::vector<zmq::message_t>& parts) {
}
}
void LokiMQ::proxy_run_worker(run_info& run) {
void OxenMQ::proxy_run_worker(run_info& run) {
if (!run.worker_thread.joinable())
run.worker_thread = std::thread{[this, id=run.worker_id] { worker_thread(id); }};
else
send_routed_message(workers_socket, run.worker_routing_id, "RUN");
}
void LokiMQ::proxy_to_worker(size_t conn_index, std::vector<zmq::message_t>& parts) {
void OxenMQ::proxy_to_worker(size_t conn_index, std::vector<zmq::message_t>& parts) {
bool outgoing = connections[conn_index].get(zmq::sockopt::type) == ZMQ_DEALER;
peer_info tmp_peer;
@ -377,7 +377,7 @@ void LokiMQ::proxy_to_worker(size_t conn_index, std::vector<zmq::message_t>& par
category.active_threads++;
}
void LokiMQ::inject_task(const std::string& category, std::string command, std::string remote, std::function<void()> callback) {
void OxenMQ::inject_task(const std::string& category, std::string command, std::string remote, std::function<void()> callback) {
if (!callback) return;
auto it = categories.find(category);
if (it == categories.end())
@ -386,7 +386,7 @@ void LokiMQ::inject_task(const std::string& category, std::string command, std::
injected_task{it->second, std::move(command), std::move(remote), std::move(callback)})));
}
void LokiMQ::proxy_inject_task(injected_task task) {
void OxenMQ::proxy_inject_task(injected_task task) {
auto& category = task.cat;
if (category.active_threads >= category.reserved_threads && active_workers() >= general_workers) {
// No free worker slot, queue for later

2
tests/CMakeLists.txt
View File

@ -19,7 +19,7 @@ add_executable(tests ${LMQ_TEST_SRC})
find_package(Threads)
target_link_libraries(tests Catch2::Catch2 lokimq Threads::Threads)
target_link_libraries(tests Catch2::Catch2 oxenmq Threads::Threads)
set_target_properties(tests PROPERTIES
CXX_STANDARD 17

6
tests/common.h
View File

@ -1,8 +1,8 @@
#pragma once
#include "lokimq/lokimq.h"
#include "oxenmq/oxenmq.h"
#include <catch2/catch.hpp>
using namespace lokimq;
using namespace oxenmq;
static auto startup = std::chrono::steady_clock::now();
@ -41,7 +41,7 @@ inline std::unique_lock<std::mutex> catch_lock() {
return std::unique_lock<std::mutex>{mutex};
}
inline LokiMQ::Logger get_logger(std::string prefix = "") {
inline OxenMQ::Logger get_logger(std::string prefix = "") {
std::string me = "tests/common.h";
std::string strip = __FILE__;
if (strip.substr(strip.size() - me.size()) == me)

2
tests/test_address.cpp
View File

@ -1,4 +1,4 @@
#include "lokimq/address.h"
#include "oxenmq/address.h"
#include "common.h"
const std::string pk = "\xf1\x6b\xa5\x59\x10\x39\xf0\x89\xb4\x2a\x83\x41\x75\x09\x30\x94\x07\x4d\x0d\x93\x7a\x79\xe5\x3e\x5c\xe7\x30\xf9\x46\xe1\x4b\x88";

18
tests/test_batch.cpp
View File

@ -1,4 +1,4 @@
#include "lokimq/batch.h"
#include "oxenmq/batch.h"
#include "common.h"
#include <future>
@ -12,7 +12,7 @@ double do_my_task(int input) {
std::promise<std::pair<double, int>> done;
void continue_big_task(std::vector<lokimq::job_result<double>> results) {
void continue_big_task(std::vector<oxenmq::job_result<double>> results) {
double sum = 0;
int exc_count = 0;
for (auto& r : results) {
@ -25,10 +25,10 @@ void continue_big_task(std::vector<lokimq::job_result<double>> results) {
done.set_value({sum, exc_count});
}
void start_big_task(lokimq::LokiMQ& lmq) {
void start_big_task(oxenmq::OxenMQ& lmq) {
size_t num_jobs = 32;
lokimq::Batch<double /*return type*/> batch;
oxenmq::Batch<double /*return type*/> batch;
batch.reserve(num_jobs);
for (size_t i = 0; i < num_jobs; i++)
@ -41,7 +41,7 @@ void start_big_task(lokimq::LokiMQ& lmq) {
TEST_CASE("batching many small jobs", "[batch-many]") {
lokimq::LokiMQ lmq{
oxenmq::OxenMQ lmq{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -58,7 +58,7 @@ TEST_CASE("batching many small jobs", "[batch-many]") {
}
TEST_CASE("batch exception propagation", "[batch-exceptions]") {
lokimq::LokiMQ lmq{
oxenmq::OxenMQ lmq{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -73,7 +73,7 @@ TEST_CASE("batch exception propagation", "[batch-exceptions]") {
using Catch::Matchers::Message;
SECTION( "value return" ) {
lokimq::Batch<int> batch;
oxenmq::Batch<int> batch;
for (int i : {1, 2})
batch.add_job([i]() { if (i == 1) return 42; throw std::domain_error("bad value " + std::to_string(i)); });
batch.completion([&done_promise](auto results) {
@ -88,7 +88,7 @@ TEST_CASE("batch exception propagation", "[batch-exceptions]") {
}
SECTION( "lvalue return" ) {
lokimq::Batch<int&> batch;
oxenmq::Batch<int&> batch;
int forty_two = 42;
for (int i : {1, 2})
batch.add_job([i,&forty_two]() -> int& {
@ -110,7 +110,7 @@ TEST_CASE("batch exception propagation", "[batch-exceptions]") {
}
SECTION( "void return" ) {
lokimq::Batch<void> batch;
oxenmq::Batch<void> batch;
for (int i : {1, 2})
batch.add_job([i]() { if (i != 1) throw std::domain_error("bad value " + std::to_string(i)); });
batch.completion([&done_promise](auto results) {

16
tests/test_bt.cpp
View File

@ -1,4 +1,4 @@
#include "lokimq/bt_serialize.h"
#include "oxenmq/bt_serialize.h"
#include "common.h"
#include <map>
#include <set>
@ -129,10 +129,10 @@ TEST_CASE("bt_value deserialization", "[bt][deserialization][bt_value]") {
REQUIRE( var::get<int64_t>(dna2) == -42 );
REQUIRE_THROWS( var::get<int64_t>(dna1) );
REQUIRE_THROWS( var::get<uint64_t>(dna2) );
REQUIRE( lokimq::get_int<int>(dna1) == 42 );
REQUIRE( lokimq::get_int<int>(dna2) == -42 );
REQUIRE( lokimq::get_int<unsigned>(dna1) == 42 );
REQUIRE_THROWS( lokimq::get_int<unsigned>(dna2) );
REQUIRE( oxenmq::get_int<int>(dna1) == 42 );
REQUIRE( oxenmq::get_int<int>(dna2) == -42 );
REQUIRE( oxenmq::get_int<unsigned>(dna1) == 42 );
REQUIRE_THROWS( oxenmq::get_int<unsigned>(dna2) );
bt_value x = bt_deserialize<bt_value>("d3:barle3:foold1:ali1ei2ei3ee1:bleed1:cli-5ei4eeeee");
REQUIRE( std::holds_alternative<bt_dict>(x) );
@ -150,9 +150,9 @@ TEST_CASE("bt_value deserialization", "[bt][deserialization][bt_value]") {
bt_list& foo1b = var::get<bt_list>(foo1.at("b"));
bt_list& foo2c = var::get<bt_list>(foo2.at("c"));
std::list<int> foo1a_vals, foo1b_vals, foo2c_vals;
for (auto& v : foo1a) foo1a_vals.push_back(lokimq::get_int<int>(v));
for (auto& v : foo1b) foo1b_vals.push_back(lokimq::get_int<int>(v));
for (auto& v : foo2c) foo2c_vals.push_back(lokimq::get_int<int>(v));
for (auto& v : foo1a) foo1a_vals.push_back(oxenmq::get_int<int>(v));
for (auto& v : foo1b) foo1b_vals.push_back(oxenmq::get_int<int>(v));
for (auto& v : foo2c) foo2c_vals.push_back(oxenmq::get_int<int>(v));
REQUIRE( foo1a_vals == std::list{{1,2,3}} );
REQUIRE( foo1b_vals == std::list<int>{} );
REQUIRE( foo2c_vals == std::list{{-5, 4}} );

36
tests/test_commands.cpp
View File

@ -1,13 +1,13 @@
#include "common.h"
#include <lokimq/hex.h>
#include <oxenmq/hex.h>
#include <map>
#include <set>
using namespace lokimq;
using namespace oxenmq;
TEST_CASE("basic commands", "[commands]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -31,7 +31,7 @@ TEST_CASE("basic commands", "[commands]") {
server.start();
LokiMQ client{get_logger("C» "), LogLevel::trace};
OxenMQ client{get_logger("C» "), LogLevel::trace};
client.add_category("public", Access{AuthLevel::none});
client.add_command("public", "hi", [&](auto&) { his++; });
@ -77,7 +77,7 @@ TEST_CASE("basic commands", "[commands]") {
TEST_CASE("outgoing auth level", "[commands][auth]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -93,7 +93,7 @@ TEST_CASE("outgoing auth level", "[commands][auth]") {
server.start();
LokiMQ client{get_logger("C» "), LogLevel::trace};
OxenMQ client{get_logger("C» "), LogLevel::trace};
std::atomic<int> public_hi{0}, basic_hi{0}, admin_hi{0};
client.add_category("public", Access{AuthLevel::none});
@ -159,7 +159,7 @@ TEST_CASE("deferred replies on incoming connections", "[commands][hey google]")
// original node.
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -178,7 +178,7 @@ TEST_CASE("deferred replies on incoming connections", "[commands][hey google]")
m.send_reply("Okay, I'll remember that.");
if (backdoor)
m.lokimq.send(backdoor, "backdoor.data", m.data[0]);
m.oxenmq.send(backdoor, "backdoor.data", m.data[0]);
});
server.add_command("hey google", "recall", [&](Message& m) {
auto l = catch_lock();
@ -199,7 +199,7 @@ TEST_CASE("deferred replies on incoming connections", "[commands][hey google]")
std::set<std::string> backdoor_details;
LokiMQ nsa{get_logger("NSA» ")};
OxenMQ nsa{get_logger("NSA» ")};
nsa.add_category("backdoor", Access{AuthLevel::admin});
nsa.add_command("backdoor", "data", [&](Message& m) {
auto l = catch_lock();
@ -215,7 +215,7 @@ TEST_CASE("deferred replies on incoming connections", "[commands][hey google]")
REQUIRE( backdoor );
}
std::vector<std::unique_ptr<LokiMQ>> clients;
std::vector<std::unique_ptr<OxenMQ>> clients;
std::vector<ConnectionID> conns;
std::map<int, std::set<std::string>> personal_details{
{0, {"Loretta"s, "photos"s}},
@ -231,7 +231,7 @@ TEST_CASE("deferred replies on incoming connections", "[commands][hey google]")
std::map<int, std::set<std::string>> google_knows;
int things_remembered{0};
for (int i = 0; i < 5; i++) {
clients.push_back(std::make_unique<LokiMQ>(
clients.push_back(std::make_unique<OxenMQ>(
get_logger("C" + std::to_string(i) + "» "), LogLevel::trace
));
auto& c = clients.back();
@ -271,7 +271,7 @@ TEST_CASE("deferred replies on incoming connections", "[commands][hey google]")
TEST_CASE("send failure callbacks", "[commands][queue_full]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -298,7 +298,7 @@ TEST_CASE("send failure callbacks", "[commands][queue_full]") {
server.start();
// Use a raw socket here because I want to stall it by not reading from it at all, and that is
// hard with LokiMQ.
// hard with OxenMQ.
zmq::context_t client_ctx;
zmq::socket_t client{client_ctx, zmq::socket_type::dealer};
client.connect(listen);
@ -365,7 +365,7 @@ TEST_CASE("send failure callbacks", "[commands][queue_full]") {
TEST_CASE("data parts", "[send][data_parts]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -385,7 +385,7 @@ TEST_CASE("data parts", "[send][data_parts]") {
});
server.start();
LokiMQ client{get_logger("C» "), LogLevel::trace};
OxenMQ client{get_logger("C» "), LogLevel::trace};
client.start();
std::atomic<bool> got{false};
@ -406,7 +406,7 @@ TEST_CASE("data parts", "[send][data_parts]") {
}
std::vector some_data{{"abc"s, "def"s, "omg123\0zzz"s}};
client.send(c, "public.hello", lokimq::send_option::data_parts(some_data.begin(), some_data.end()));
client.send(c, "public.hello", oxenmq::send_option::data_parts(some_data.begin(), some_data.end()));
reply_sleep();
{
auto lock = catch_lock();
@ -418,10 +418,10 @@ TEST_CASE("data parts", "[send][data_parts]") {
std::vector some_data2{{"a"sv, "b"sv, "\0"sv}};
client.send(c, "public.hello",
"hi",
lokimq::send_option::data_parts(some_data2.begin(), some_data2.end()),
oxenmq::send_option::data_parts(some_data2.begin(), some_data2.end()),
"another",
"string"sv,
lokimq::send_option::data_parts(some_data.begin(), some_data.end()));
oxenmq::send_option::data_parts(some_data.begin(), some_data.end()));
std::vector<std::string> expected;
expected.push_back("hi");

30
tests/test_connect.cpp
View File

@ -1,5 +1,5 @@
#include "common.h"
#include <lokimq/hex.h>
#include <oxenmq/hex.h>
extern "C" {
#include <sodium.h>
}
@ -7,7 +7,7 @@ extern "C" {
TEST_CASE("connections with curve authentication", "[curve][connect]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -20,7 +20,7 @@ TEST_CASE("connections with curve authentication", "[curve][connect]") {
server.add_request_command("public", "hello", [&](Message& m) { m.send_reply("hi"); });
server.start();
LokiMQ client{get_logger("C» "), LogLevel::trace};
OxenMQ client{get_logger("C» "), LogLevel::trace};
client.start();
@ -55,7 +55,7 @@ TEST_CASE("self-connection SN optimization", "[connect][self]") {
REQUIRE(sodium_init() != -1);
auto listen_addr = random_localhost();
crypto_box_keypair(reinterpret_cast<unsigned char*>(&pubkey[0]), reinterpret_cast<unsigned char*>(&privkey[0]));
LokiMQ sn{
OxenMQ sn{
pubkey, privkey,
true,
[&](auto pk) { if (pk == pubkey) return listen_addr; else return ""s; },
@ -92,7 +92,7 @@ TEST_CASE("self-connection SN optimization", "[connect][self]") {
TEST_CASE("plain-text connections", "[plaintext][connect]") {
std::string listen = random_localhost();
LokiMQ server{get_logger("S» "), LogLevel::trace};
OxenMQ server{get_logger("S» "), LogLevel::trace};
server.add_category("public", Access{AuthLevel::none});
server.add_request_command("public", "hello", [&](Message& m) { m.send_reply("hi"); });
@ -101,7 +101,7 @@ TEST_CASE("plain-text connections", "[plaintext][connect]") {
server.start();
LokiMQ client{get_logger("C» "), LogLevel::trace};
OxenMQ client{get_logger("C» "), LogLevel::trace};
client.start();
@ -131,7 +131,7 @@ TEST_CASE("plain-text connections", "[plaintext][connect]") {
TEST_CASE("unique connection IDs", "[connect][id]") {
std::string listen = random_localhost();
LokiMQ server{get_logger("S» "), LogLevel::trace};
OxenMQ server{get_logger("S» "), LogLevel::trace};
ConnectionID first, second;
server.add_category("x", Access{AuthLevel::none})
@ -143,8 +143,8 @@ TEST_CASE("unique connection IDs", "[connect][id]") {
server.start();
LokiMQ client1{get_logger("C1» "), LogLevel::trace};
LokiMQ client2{get_logger("C2» "), LogLevel::trace};
OxenMQ client1{get_logger("C1» "), LogLevel::trace};
OxenMQ client2{get_logger("C2» "), LogLevel::trace};
client1.start();
client2.start();
@ -186,7 +186,7 @@ TEST_CASE("unique connection IDs", "[connect][id]") {
TEST_CASE("SN disconnections", "[connect][disconnect]") {
std::vector<std::unique_ptr<LokiMQ>> lmq;
std::vector<std::unique_ptr<OxenMQ>> lmq;
std::vector<std::string> pubkey, privkey;
std::unordered_map<std::string, std::string> conn;
REQUIRE(sodium_init() != -1);
@ -200,7 +200,7 @@ TEST_CASE("SN disconnections", "[connect][disconnect]") {
}
std::atomic<int> his{0};
for (int i = 0; i < pubkey.size(); i++) {
lmq.push_back(std::make_unique<LokiMQ>(
lmq.push_back(std::make_unique<OxenMQ>(
pubkey[i], privkey[i], true,
[conn](auto pk) { auto it = conn.find((std::string) pk); if (it != conn.end()) return it->second; return ""s; },
get_logger("S" + std::to_string(i) + "» "),
@ -238,7 +238,7 @@ TEST_CASE("SN auth checks", "[sandwich][auth]") {
privkey.resize(crypto_box_SECRETKEYBYTES);
REQUIRE(sodium_init() != -1);
crypto_box_keypair(reinterpret_cast<unsigned char*>(&pubkey[0]), reinterpret_cast<unsigned char*>(&privkey[0]));
LokiMQ server{
OxenMQ server{
pubkey, privkey,
true, // service node
[](auto) { return ""; },
@ -265,7 +265,7 @@ TEST_CASE("SN auth checks", "[sandwich][auth]") {
.add_request_command("make", [&](Message& m) { m.send_reply("okay"); });
server.start();
LokiMQ client{
OxenMQ client{
"", "", false,
[&](auto remote_pk) { if (remote_pk == pubkey) return listen; return ""s; },
get_logger("B» "), LogLevel::trace};
@ -352,7 +352,7 @@ TEST_CASE("SN single worker test", "[connect][worker]") {
// Tests a failure case that could trigger when all workers are allocated (here we make that
// simpler by just having one worker).
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "",
false, // service node
[](auto) { return ""; },
@ -368,7 +368,7 @@ TEST_CASE("SN single worker test", "[connect][worker]") {
;
server.start();
LokiMQ client{get_logger("B» "), LogLevel::trace};
OxenMQ client{get_logger("B» "), LogLevel::trace};
client.start();
auto conn = client.connect_remote(listen, [](auto) {}, [](auto, auto) {});

213
tests/test_encoding.cpp
View File

@ -1,7 +1,8 @@
#include "lokimq/hex.h"
#include "lokimq/base32z.h"
#include "lokimq/base64.h"
#include "oxenmq/hex.h"
#include "oxenmq/base32z.h"
#include "oxenmq/base64.h"
#include "common.h"
#include <iterator>
using namespace std::literals;
@ -11,116 +12,123 @@ const std::string pk_b32z = "6fi4kseo88aeupbkopyzknjo1odw4dcuxjh6kx1hhhax1tzbjqr
const std::string pk_b64 = "8WulWRA58Im0KoNBdQkwlAdNDZN6eeU+XOcw+UbhS4g=";
TEST_CASE("hex encoding/decoding", "[encoding][decoding][hex]") {
REQUIRE( lokimq::to_hex("\xff\x42\x12\x34") == "ff421234"s );
REQUIRE( oxenmq::to_hex("\xff\x42\x12\x34") == "ff421234"s );
std::vector<uint8_t> chars{{1, 10, 100, 254}};
std::array<uint8_t, 8> out;
std::array<uint8_t, 8> expected{{'0', '1', '0', 'a', '6', '4', 'f', 'e'}};
lokimq::to_hex(chars.begin(), chars.end(), out.begin());
oxenmq::to_hex(chars.begin(), chars.end(), out.begin());
REQUIRE( out == expected );
REQUIRE( lokimq::to_hex(chars.begin(), chars.end()) == "010a64fe" );
REQUIRE( oxenmq::to_hex(chars.begin(), chars.end()) == "010a64fe" );
REQUIRE( lokimq::from_hex("12345678ffEDbca9") == "\x12\x34\x56\x78\xff\xed\xbc\xa9"s );
REQUIRE( oxenmq::from_hex("12345678ffEDbca9") == "\x12\x34\x56\x78\xff\xed\xbc\xa9"s );
REQUIRE( lokimq::is_hex("1234567890abcdefABCDEF1234567890abcdefABCDEF") );
REQUIRE_FALSE( lokimq::is_hex("1234567890abcdefABCDEF1234567890aGcdefABCDEF") );
REQUIRE_FALSE( lokimq::is_hex("1234567890abcdefABCDEF1234567890agcdefABCDEF") );
REQUIRE_FALSE( lokimq::is_hex("\x11\xff") );
REQUIRE( oxenmq::is_hex("1234567890abcdefABCDEF1234567890abcdefABCDEF") );
REQUIRE_FALSE( oxenmq::is_hex("1234567890abcdefABCDEF1234567890aGcdefABCDEF") );
// ^
REQUIRE_FALSE( oxenmq::is_hex("1234567890abcdefABCDEF1234567890agcdefABCDEF") );
// ^
REQUIRE_FALSE( oxenmq::is_hex("\x11\xff") );
constexpr auto odd_hex = "1234567890abcdefABCDEF1234567890abcdefABCDE"sv;
REQUIRE_FALSE( oxenmq::is_hex(odd_hex) );
REQUIRE_FALSE( oxenmq::is_hex("0") );
REQUIRE( lokimq::from_hex(pk_hex) == pk );
REQUIRE( lokimq::to_hex(pk) == pk_hex );
REQUIRE( std::all_of(odd_hex.begin(), odd_hex.end(), oxenmq::is_hex_digit<char>) );
REQUIRE( lokimq::from_hex(pk_hex.begin(), pk_hex.end()) == pk );
REQUIRE( oxenmq::from_hex(pk_hex) == pk );
REQUIRE( oxenmq::to_hex(pk) == pk_hex );
REQUIRE( oxenmq::from_hex(pk_hex.begin(), pk_hex.end()) == pk );
std::vector<std::byte> bytes{{std::byte{0xff}, std::byte{0x42}, std::byte{0x12}, std::byte{0x34}}};
std::basic_string_view<std::byte> b{bytes.data(), bytes.size()};
REQUIRE( lokimq::to_hex(b) == "ff421234"s );
REQUIRE( oxenmq::to_hex(b) == "ff421234"s );
bytes.resize(8);
bytes[0] = std::byte{'f'}; bytes[1] = std::byte{'f'}; bytes[2] = std::byte{'4'}; bytes[3] = std::byte{'2'};
bytes[4] = std::byte{'1'}; bytes[5] = std::byte{'2'}; bytes[6] = std::byte{'3'}; bytes[7] = std::byte{'4'};
std::basic_string_view<std::byte> hex_bytes{bytes.data(), bytes.size()};
REQUIRE( lokimq::is_hex(hex_bytes) );
REQUIRE( lokimq::from_hex(hex_bytes) == "\xff\x42\x12\x34" );
REQUIRE( oxenmq::is_hex(hex_bytes) );
REQUIRE( oxenmq::from_hex(hex_bytes) == "\xff\x42\x12\x34" );
}
TEST_CASE("base32z encoding/decoding", "[encoding][decoding][base32z]") {
REQUIRE( lokimq::to_base32z("\0\0\0\0\0"s) == "yyyyyyyy" );
REQUIRE( lokimq::to_base32z("\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef"sv)
REQUIRE( oxenmq::to_base32z("\0\0\0\0\0"s) == "yyyyyyyy" );
REQUIRE( oxenmq::to_base32z("\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef"sv)
== "yrtwk3hjixg66yjdeiuauk6p7hy1gtm8tgih55abrpnsxnpm3zzo");
REQUIRE( lokimq::from_base32z("yrtwk3hjixg66yjdeiuauk6p7hy1gtm8tgih55abrpnsxnpm3zzo")
REQUIRE( oxenmq::from_base32z("yrtwk3hjixg66yjdeiuauk6p7hy1gtm8tgih55abrpnsxnpm3zzo")
== "\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef"sv);
REQUIRE( lokimq::from_base32z("YRTWK3HJIXG66YJDEIUAUK6P7HY1GTM8TGIH55ABRPNSXNPM3ZZO")
REQUIRE( oxenmq::from_base32z("YRTWK3HJIXG66YJDEIUAUK6P7HY1GTM8TGIH55ABRPNSXNPM3ZZO")
== "\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef"sv);
auto five_nulls = lokimq::from_base32z("yyyyyyyy");
auto five_nulls = oxenmq::from_base32z("yyyyyyyy");
REQUIRE( five_nulls.size() == 5 );
REQUIRE( five_nulls == "\0\0\0\0\0"s );
// 00000 00001 00010 00011 00100 00101 00110 00111
// ==
// 00000000 01000100 00110010 00010100 11000111
REQUIRE( lokimq::from_base32z("ybndrfg8") == "\x00\x44\x32\x14\xc7"s );
REQUIRE( oxenmq::from_base32z("ybndrfg8") == "\x00\x44\x32\x14\xc7"s );
// Special case 1: 7 base32z digits with 3 trailing 0 bits -> 4 bytes (the trailing 0s are dropped)
// 00000 00001 00010 00011 00100 00101 11000
// ==
// 00000000 01000100 00110010 00010111
REQUIRE( lokimq::from_base32z("ybndrfa") == "\x00\x44\x32\x17"s );
REQUIRE( oxenmq::from_base32z("ybndrfa") == "\x00\x44\x32\x17"s );
// Round-trip it:
REQUIRE( lokimq::from_base32z(lokimq::to_base32z("\x00\x44\x32\x17"sv)) == "\x00\x44\x32\x17"sv );
REQUIRE( lokimq::to_base32z(lokimq::from_base32z("ybndrfa")) == "ybndrfa" );
REQUIRE( oxenmq::from_base32z(oxenmq::to_base32z("\x00\x44\x32\x17"sv)) == "\x00\x44\x32\x17"sv );
REQUIRE( oxenmq::to_base32z(oxenmq::from_base32z("ybndrfa")) == "ybndrfa" );
// Special case 2: 7 base32z digits with 3 trailing bits 010; we just ignore the trailing stuff,
// as if it was specified as 0. (The last digit here is 11010 instead of 11000).
REQUIRE( lokimq::from_base32z("ybndrf4") == "\x00\x44\x32\x17"s );
REQUIRE( oxenmq::from_base32z("ybndrf4") == "\x00\x44\x32\x17"s );
// This one won't round-trip to the same value since it has ignored garbage bytes at the end
REQUIRE( lokimq::to_base32z(lokimq::from_base32z("ybndrf4"s)) == "ybndrfa" );
REQUIRE( oxenmq::to_base32z(oxenmq::from_base32z("ybndrf4"s)) == "ybndrfa" );
REQUIRE( lokimq::to_base32z(pk) == pk_b32z );
REQUIRE( lokimq::to_base32z(pk.begin(), pk.end()) == pk_b32z );
REQUIRE( lokimq::from_base32z(pk_b32z) == pk );
REQUIRE( lokimq::from_base32z(pk_b32z.begin(), pk_b32z.end()) == pk );
REQUIRE( oxenmq::to_base32z(pk) == pk_b32z );
REQUIRE( oxenmq::to_base32z(pk.begin(), pk.end()) == pk_b32z );
REQUIRE( oxenmq::from_base32z(pk_b32z) == pk );
REQUIRE( oxenmq::from_base32z(pk_b32z.begin(), pk_b32z.end()) == pk );
std::string pk_b32z_again, pk_again;
lokimq::to_base32z(pk.begin(), pk.end(), std::back_inserter(pk_b32z_again));
lokimq::from_base32z(pk_b32z.begin(), pk_b32z.end(), std::back_inserter(pk_again));
oxenmq::to_base32z(pk.begin(), pk.end(), std::back_inserter(pk_b32z_again));
oxenmq::from_base32z(pk_b32z.begin(), pk_b32z.end(), std::back_inserter(pk_again));
REQUIRE( pk_b32z_again == pk_b32z );
REQUIRE( pk_again == pk );
std::vector<std::byte> bytes{{std::byte{0}, std::byte{255}}};
std::basic_string_view<std::byte> b{bytes.data(), bytes.size()};
REQUIRE( lokimq::to_base32z(b) == "yd9o" );
REQUIRE( oxenmq::to_base32z(b) == "yd9o" );
bytes.resize(4);
bytes[0] = std::byte{'y'}; bytes[1] = std::byte{'d'}; bytes[2] = std::byte{'9'}; bytes[3] = std::byte{'o'};
std::basic_string_view<std::byte> b32_bytes{bytes.data(), bytes.size()};
REQUIRE( lokimq::is_base32z(b32_bytes) );
REQUIRE( lokimq::from_base32z(b32_bytes) == "\x00\xff"sv );
REQUIRE( oxenmq::is_base32z(b32_bytes) );
REQUIRE( oxenmq::from_base32z(b32_bytes) == "\x00\xff"sv );
}
TEST_CASE("base64 encoding/decoding", "[encoding][decoding][base64]") {
// 00000000 00000000 00000000 -> 000000 000000 000000 000000
REQUIRE( lokimq::to_base64("\0\0\0"s) == "AAAA" );
REQUIRE( oxenmq::to_base64("\0\0\0"s) == "AAAA" );
// 00000001 00000002 00000003 -> 000000 010000 000200 000003
REQUIRE( lokimq::to_base64("\x01\x02\x03"s) == "AQID" );
REQUIRE( lokimq::to_base64("\0\0\0\0"s) == "AAAAAA==" );
REQUIRE( oxenmq::to_base64("\x01\x02\x03"s) == "AQID" );
REQUIRE( oxenmq::to_base64("\0\0\0\0"s) == "AAAAAA==" );
// 00000000 00000000 00000000 11111111 ->
// 000000 000000 000000 000000 111111 110000 (pad) (pad)
REQUIRE( lokimq::to_base64("a") == "YQ==" );
REQUIRE( lokimq::to_base64("ab") == "YWI=" );
REQUIRE( lokimq::to_base64("abc") == "YWJj" );
REQUIRE( lokimq::to_base64("abcd") == "YWJjZA==" );
REQUIRE( lokimq::to_base64("abcde") == "YWJjZGU=" );
REQUIRE( lokimq::to_base64("abcdef") == "YWJjZGVm" );
REQUIRE( oxenmq::to_base64("a") == "YQ==" );
REQUIRE( oxenmq::to_base64("ab") == "YWI=" );
REQUIRE( oxenmq::to_base64("abc") == "YWJj" );
REQUIRE( oxenmq::to_base64("abcd") == "YWJjZA==" );
REQUIRE( oxenmq::to_base64("abcde") == "YWJjZGU=" );
REQUIRE( oxenmq::to_base64("abcdef") == "YWJjZGVm" );
REQUIRE( lokimq::to_base64("\0\0\0\xff"s) == "AAAA/w==" );
REQUIRE( lokimq::to_base64("\0\0\0\xff\xff"s) == "AAAA//8=" );
REQUIRE( lokimq::to_base64("\0\0\0\xff\xff\xff"s) == "AAAA////" );
REQUIRE( lokimq::to_base64(
REQUIRE( oxenmq::to_base64("\0\0\0\xff"s) == "AAAA/w==" );
REQUIRE( oxenmq::to_base64("\0\0\0\xff\xff"s) == "AAAA//8=" );
REQUIRE( oxenmq::to_base64("\0\0\0\xff\xff\xff"s) == "AAAA////" );
REQUIRE( oxenmq::to_base64(
"Man is distinguished, not only by his reason, but by this singular passion from other "
"animals, which is a lust of the mind, that by a perseverance of delight in the "
"continued and indefatigable generation of knowledge, exceeds the short vehemence of "
@ -132,33 +140,33 @@ TEST_CASE("base64 encoding/decoding", "[encoding][decoding][base64]") {
"dWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdlLCBleGNlZWRzIHRo"
"ZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4=" );
REQUIRE( lokimq::from_base64("A+/A") == "\x03\xef\xc0" );
REQUIRE( lokimq::from_base64("YWJj") == "abc" );
REQUIRE( lokimq::from_base64("YWJjZA==") == "abcd" );
REQUIRE( lokimq::from_base64("YWJjZA") == "abcd" );
REQUIRE( lokimq::from_base64("YWJjZB") == "abcd" ); // ignore superfluous bits
REQUIRE( lokimq::from_base64("YWJjZB") == "abcd" ); // ignore superfluous bits
REQUIRE( lokimq::from_base64("YWJj+") == "abc" ); // ignore superfluous bits
REQUIRE( lokimq::from_base64("YWJjZGU=") == "abcde" );
REQUIRE( lokimq::from_base64("YWJjZGU") == "abcde" );
REQUIRE( lokimq::from_base64("YWJjZGVm") == "abcdef" );
REQUIRE( oxenmq::from_base64("A+/A") == "\x03\xef\xc0" );
REQUIRE( oxenmq::from_base64("YWJj") == "abc" );
REQUIRE( oxenmq::from_base64("YWJjZA==") == "abcd" );
REQUIRE( oxenmq::from_base64("YWJjZA") == "abcd" );
REQUIRE( oxenmq::from_base64("YWJjZB") == "abcd" ); // ignore superfluous bits
REQUIRE( oxenmq::from_base64("YWJjZB") == "abcd" ); // ignore superfluous bits
REQUIRE( oxenmq::from_base64("YWJj+") == "abc" ); // ignore superfluous bits
REQUIRE( oxenmq::from_base64("YWJjZGU=") == "abcde" );
REQUIRE( oxenmq::from_base64("YWJjZGU") == "abcde" );
REQUIRE( oxenmq::from_base64("YWJjZGVm") == "abcdef" );
REQUIRE( lokimq::is_base64("YWJjZGVm") );
REQUIRE( lokimq::is_base64("YWJjZGU") );
REQUIRE( lokimq::is_base64("YWJjZGU=") );
REQUIRE( lokimq::is_base64("YWJjZA==") );
REQUIRE( lokimq::is_base64("YWJjZA") );
REQUIRE( lokimq::is_base64("YWJjZB") ); // not really valid, but we explicitly accept it
REQUIRE( oxenmq::is_base64("YWJjZGVm") );
REQUIRE( oxenmq::is_base64("YWJjZGU") );
REQUIRE( oxenmq::is_base64("YWJjZGU=") );
REQUIRE( oxenmq::is_base64("YWJjZA==") );
REQUIRE( oxenmq::is_base64("YWJjZA") );
REQUIRE( oxenmq::is_base64("YWJjZB") ); // not really valid, but we explicitly accept it
REQUIRE_FALSE( lokimq::is_base64("YWJjZ=") ); // invalid padding (padding can only be 4th or 3rd+4th of a 4-char block)
REQUIRE_FALSE( lokimq::is_base64("YWJj=") );
REQUIRE_FALSE( lokimq::is_base64("YWJj=A") );
REQUIRE_FALSE( lokimq::is_base64("YWJjA===") );
REQUIRE_FALSE( lokimq::is_base64("YWJ[") );
REQUIRE_FALSE( lokimq::is_base64("YWJ.") );
REQUIRE_FALSE( lokimq::is_base64("_YWJ") );
REQUIRE_FALSE( oxenmq::is_base64("YWJjZ=") ); // invalid padding (padding can only be 4th or 3rd+4th of a 4-char block)
REQUIRE_FALSE( oxenmq::is_base64("YWJj=") );
REQUIRE_FALSE( oxenmq::is_base64("YWJj=A") );
REQUIRE_FALSE( oxenmq::is_base64("YWJjA===") );
REQUIRE_FALSE( oxenmq::is_base64("YWJ[") );
REQUIRE_FALSE( oxenmq::is_base64("YWJ.") );
REQUIRE_FALSE( oxenmq::is_base64("_YWJ") );
REQUIRE( lokimq::from_base64(
REQUIRE( oxenmq::from_base64(
"TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0aGlz"
"IHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1c3Qgb2Yg"
"dGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0aGUgY29udGlu"
@ -170,24 +178,63 @@ TEST_CASE("base64 encoding/decoding", "[encoding][decoding][base64]") {
"continued and indefatigable generation of knowledge, exceeds the short vehemence of "
"any carnal pleasure.");
REQUIRE( lokimq::to_base64(pk) == pk_b64 );
REQUIRE( lokimq::to_base64(pk.begin(), pk.end()) == pk_b64 );
REQUIRE( lokimq::from_base64(pk_b64) == pk );
REQUIRE( lokimq::from_base64(pk_b64.begin(), pk_b64.end()) == pk );
REQUIRE( oxenmq::to_base64(pk) == pk_b64 );
REQUIRE( oxenmq::to_base64(pk.begin(), pk.end()) == pk_b64 );
REQUIRE( oxenmq::from_base64(pk_b64) == pk );
REQUIRE( oxenmq::from_base64(pk_b64.begin(), pk_b64.end()) == pk );
std::string pk_b64_again, pk_again;
lokimq::to_base64(pk.begin(), pk.end(), std::back_inserter(pk_b64_again));
lokimq::from_base64(pk_b64.begin(), pk_b64.end(), std::back_inserter(pk_again));
oxenmq::to_base64(pk.begin(), pk.end(), std::back_inserter(pk_b64_again));
oxenmq::from_base64(pk_b64.begin(), pk_b64.end(), std::back_inserter(pk_again));
REQUIRE( pk_b64_again == pk_b64 );
REQUIRE( pk_again == pk );
std::vector<std::byte> bytes{{std::byte{0}, std::byte{255}}};
std::basic_string_view<std::byte> b{bytes.data(), bytes.size()};
REQUIRE( lokimq::to_base64(b) == "AP8=" );
REQUIRE( oxenmq::to_base64(b) == "AP8=" );
bytes.resize(4);
bytes[0] = std::byte{'/'}; bytes[1] = std::byte{'w'}; bytes[2] = std::byte{'A'}; bytes[3] = std::byte{'='};
std::basic_string_view<std::byte> b64_bytes{bytes.data(), bytes.size()};
REQUIRE( lokimq::is_base64(b64_bytes) );
REQUIRE( lokimq::from_base64(b64_bytes) == "\xff\x00"sv );
REQUIRE( oxenmq::is_base64(b64_bytes) );
REQUIRE( oxenmq::from_base64(b64_bytes) == "\xff\x00"sv );
}
TEST_CASE("std::byte decoding", "[decoding][hex][base32z][base64]") {
// Decoding to std::byte is a little trickier because you can't assign to a byte without an
// explicit cast, which means we have to properly detect that output is going to a std::byte
// output.
// hex
auto b_in = "ff42"s;
std::vector<std::byte> b_out;
oxenmq::from_hex(b_in.begin(), b_in.end(), std::back_inserter(b_out));
REQUIRE( b_out == std::vector{std::byte{0xff}, std::byte{0x42}} );
b_out.emplace_back();
oxenmq::from_hex(b_in.begin(), b_in.end(), b_out.begin() + 1);
REQUIRE( b_out == std::vector{std::byte{0xff}, std::byte{0xff}, std::byte{0x42}} );
oxenmq::from_hex(b_in.begin(), b_in.end(), b_out.data());
REQUIRE( b_out == std::vector{std::byte{0xff}, std::byte{0x42}, std::byte{0x42}} );
// base32z
b_in = "yojky"s;
b_out.clear();
oxenmq::from_base32z(b_in.begin(), b_in.end(), std::back_inserter(b_out));
REQUIRE( b_out == std::vector{std::byte{0x04}, std::byte{0x12}, std::byte{0xa0}} );
b_out.emplace_back();
oxenmq::from_base32z(b_in.begin(), b_in.end(), b_out.begin() + 1);
REQUIRE( b_out == std::vector{std::byte{0x04}, std::byte{0x04}, std::byte{0x12}, std::byte{0xa0}} );
oxenmq::from_base32z(b_in.begin(), b_in.end(), b_out.data());
REQUIRE( b_out == std::vector{std::byte{0x04}, std::byte{0x12}, std::byte{0xa0}, std::byte{0xa0}} );
// base64
b_in = "yojk"s;
b_out.clear();
oxenmq::from_base64(b_in.begin(), b_in.end(), std::back_inserter(b_out));
REQUIRE( b_out == std::vector{std::byte{0xca}, std::byte{0x88}, std::byte{0xe4}} );
b_out.emplace_back();
oxenmq::from_base64(b_in.begin(), b_in.end(), b_out.begin() + 1);
REQUIRE( b_out == std::vector{std::byte{0xca}, std::byte{0xca}, std::byte{0x88}, std::byte{0xe4}} );
oxenmq::from_base64(b_in.begin(), b_in.end(), b_out.data());
REQUIRE( b_out == std::vector{std::byte{0xca}, std::byte{0x88}, std::byte{0xe4}, std::byte{0xe4}} );
}

14
tests/test_failures.cpp
View File

@ -1,13 +1,13 @@
#include "common.h"
#include <lokimq/hex.h>
#include <oxenmq/hex.h>
#include <map>
#include <set>
using namespace lokimq;
using namespace oxenmq;
TEST_CASE("failure responses - UNKNOWNCOMMAND", "[failure][UNKNOWNCOMMAND]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -48,7 +48,7 @@ TEST_CASE("failure responses - UNKNOWNCOMMAND", "[failure][UNKNOWNCOMMAND]") {
TEST_CASE("failure responses - NO_REPLY_TAG", "[failure][NO_REPLY_TAG]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -109,7 +109,7 @@ TEST_CASE("failure responses - NO_REPLY_TAG", "[failure][NO_REPLY_TAG]") {
TEST_CASE("failure responses - FORBIDDEN", "[failure][FORBIDDEN]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -192,7 +192,7 @@ TEST_CASE("failure responses - FORBIDDEN", "[failure][FORBIDDEN]") {
TEST_CASE("failure responses - NOT_A_SERVICE_NODE", "[failure][NOT_A_SERVICE_NODE]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -259,7 +259,7 @@ TEST_CASE("failure responses - NOT_A_SERVICE_NODE", "[failure][NOT_A_SERVICE_NOD
TEST_CASE("failure responses - FORBIDDEN_SN", "[failure][FORBIDDEN_SN]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },

15
tests/test_inject.cpp
View File

@ -1,10 +1,10 @@
#include "common.h"
using namespace lokimq;
using namespace oxenmq;
TEST_CASE("injected external commands", "[injected]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -24,17 +24,26 @@ TEST_CASE("injected external commands", "[injected]") {
server.start();
LokiMQ client{get_logger("C» "), LogLevel::trace};
OxenMQ client{get_logger("C» "), LogLevel::trace};
client.start();
std::atomic<bool> got{false};
bool success = false;
// Deliberately using a deprecated command here, disable -Wdeprecated-declarations
#ifdef __GNUG__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
auto c = client.connect_remote(listen,
[&](auto conn) { success = true; got = true; },
[&](auto conn, std::string_view) { got = true; },
server.get_pubkey());
#ifdef __GNUG__
#pragma GCC diagnostic pop
#endif
wait_for_conn(got);
{
auto lock = catch_lock();

20
tests/test_requests.cpp
View File

@ -1,11 +1,11 @@
#include "common.h"
#include <lokimq/hex.h>
#include <oxenmq/hex.h>
using namespace lokimq;
using namespace oxenmq;
TEST_CASE("basic requests", "[requests]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -20,7 +20,7 @@ TEST_CASE("basic requests", "[requests]") {
});
server.start();
LokiMQ client(
OxenMQ client(
[](LogLevel, const char* file, int line, std::string msg) { std::cerr << file << ":" << line << " --C-- " << msg << "\n"; }
);
//client.log_level(LogLevel::trace);
@ -62,7 +62,7 @@ TEST_CASE("basic requests", "[requests]") {
TEST_CASE("request from server to client", "[requests]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -77,7 +77,7 @@ TEST_CASE("request from server to client", "[requests]") {
});
server.start();
LokiMQ client(
OxenMQ client(
[](LogLevel, const char* file, int line, std::string msg) { std::cerr << file << ":" << line << " --C-- " << msg << "\n"; }
);
//client.log_level(LogLevel::trace);
@ -125,7 +125,7 @@ TEST_CASE("request from server to client", "[requests]") {
TEST_CASE("request timeouts", "[requests][timeout]") {
std::string listen = random_localhost();
LokiMQ server{
OxenMQ server{
"", "", // generate ephemeral keys
false, // not a service node
[](auto) { return ""; },
@ -138,7 +138,7 @@ TEST_CASE("request timeouts", "[requests][timeout]") {
server.add_request_command("public", "blackhole", [&](Message& m) { /* doesn't reply */ });
server.start();
LokiMQ client(
OxenMQ client(
[](LogLevel, const char* file, int line, std::string msg) { std::cerr << file << ":" << line << " --C-- " << msg << "\n"; }
);
//client.log_level(LogLevel::trace);
@ -167,7 +167,7 @@ TEST_CASE("request timeouts", "[requests][timeout]") {
success = ok;
data = std::move(data_);
},
lokimq::send_option::request_timeout{10ms}
oxenmq::send_option::request_timeout{10ms}
);
std::atomic<bool> got_triggered2{false};
@ -176,7 +176,7 @@ TEST_CASE("request timeouts", "[requests][timeout]") {
success = ok;
data = std::move(data_);
},
lokimq::send_option::request_timeout{200ms}
oxenmq::send_option::request_timeout{200ms}
);
std::this_thread::sleep_for(100ms);

14
tests/test_tagged_threads.cpp
View File

@ -1,9 +1,9 @@
#include "lokimq/batch.h"
#include "oxenmq/batch.h"
#include "common.h"
#include <future>
TEST_CASE("tagged thread start functions", "[tagged][start]") {
lokimq::LokiMQ lmq{get_logger(""), LogLevel::trace};
oxenmq::OxenMQ lmq{get_logger(""), LogLevel::trace};
lmq.set_general_threads(2);
lmq.set_batch_threads(2);
@ -26,13 +26,13 @@ TEST_CASE("tagged thread start functions", "[tagged][start]") {
}
TEST_CASE("tagged threads quit-before-start", "[tagged][quit]") {
auto lmq = std::make_unique<lokimq::LokiMQ>(get_logger(""), LogLevel::trace);
auto lmq = std::make_unique<oxenmq::OxenMQ>(get_logger(""), LogLevel::trace);
auto t_abc = lmq->add_tagged_thread("abc");
REQUIRE_NOTHROW(lmq.reset());
}
TEST_CASE("batch jobs to tagged threads", "[tagged][batch]") {
lokimq::LokiMQ lmq{get_logger(""), LogLevel::trace};
oxenmq::OxenMQ lmq{get_logger(""), LogLevel::trace};
lmq.set_general_threads(2);
lmq.set_batch_threads(2);
@ -111,7 +111,7 @@ TEST_CASE("batch jobs to tagged threads", "[tagged][batch]") {
}
TEST_CASE("batch job completion on tagged threads", "[tagged][batch-completion]") {
lokimq::LokiMQ lmq{get_logger(""), LogLevel::trace};
oxenmq::OxenMQ lmq{get_logger(""), LogLevel::trace};
lmq.set_general_threads(4);
lmq.set_batch_threads(4);
@ -119,7 +119,7 @@ TEST_CASE("batch job completion on tagged threads", "[tagged][batch-completion]"
auto t_abc = lmq.add_tagged_thread("abc", [&] { id_abc = std::this_thread::get_id(); });
lmq.start();
lokimq::Batch<int> batch;
oxenmq::Batch<int> batch;
for (int i = 1; i < 10; i++)
batch.add_job([i, &id_abc]() { if (std::this_thread::get_id() == id_abc) return 0; return i; });
@ -140,7 +140,7 @@ TEST_CASE("batch job completion on tagged threads", "[tagged][batch-completion]"
TEST_CASE("timer job completion on tagged threads", "[tagged][timer]") {
lokimq::LokiMQ lmq{get_logger(""), LogLevel::trace};
oxenmq::OxenMQ lmq{get_logger(""), LogLevel::trace};
lmq.set_general_threads(4);
lmq.set_batch_threads(4);