oxen-io
/
oxen-mq
mirror of https://github.com/oxen-io/oxen-mq.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #72 from jagerman/oxenc 

Use oxen-encoding and add compatibility shim headers
This commit is contained in:
Jason Rhinelander 2022-02-07 14:39:59 -04:00 committed by GitHub
parent 3d178ce3ea a0a54ed461
commit d7f5efebc1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 223 additions and 3477 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -4,3 +4,6 @@
[submodule "Catch2"]
path = tests/Catch2
url = https://github.com/catchorg/Catch2.git
[submodule "oxen-encoding"]
path = oxen-encoding
url = https://github.com/oxen-io/oxen-encoding.git

21
CMakeLists.txt
View File

@ -17,7 +17,7 @@ cmake_minimum_required(VERSION 3.7)
set(CMAKE_OSX_DEPLOYMENT_TARGET 10.12 CACHE STRING "macOS deployment target (Apple clang only)")
project(liboxenmq
VERSION 1.2.10
VERSION 1.2.11
LANGUAGES CXX C)
include(GNUInstallDirs)
@ -56,7 +56,6 @@ endif()
add_library(oxenmq
oxenmq/address.cpp
oxenmq/auth.cpp
oxenmq/bt_serialize.cpp
oxenmq/connections.cpp
oxenmq/jobs.cpp
oxenmq/oxenmq.cpp
@ -69,6 +68,24 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
target_link_libraries(oxenmq PRIVATE Threads::Threads)
if(TARGET oxenc)
target_link_libraries(oxenmq PUBLIC oxenc)
elseif(BUILD_SHARED_LIBS)
include(FindPkgConfig)
pkg_check_modules(oxenc liboxenc IMPORTED_TARGET)
if(oxenc_FOUND)
target_link_libraries(oxenmq PUBLIC PkgConfig::oxenc)
else()
add_subdirectory(oxen-encoding)
target_link_libraries(oxenmq PUBLIC oxenc)
endif()
else()
add_subdirectory(oxen-encoding)
target_link_libraries(oxenmq PUBLIC oxenc)
endif()
# libzmq is nearly impossible to link statically from a system-installed static library: it depends
# on a ton of other libraries, some of which are not all statically available. If the caller wants
# to mess with this, so be it: they can set up a libzmq target and we'll use it. Otherwise if they

1
liblokimq.pc.in
View File

@ -9,5 +9,6 @@ Version: @PROJECT_VERSION@
Libs: -L${libdir} -loxenmq
Libs.private: @PRIVATE_LIBS@
Requires: liboxenc
Requires.private: libzmq libsodium
Cflags: -I${includedir}

1
liboxenmq.pc.in
View File

@ -9,5 +9,6 @@ Version: @PROJECT_VERSION@
Libs: -L${libdir} -loxenmq
Libs.private: @PRIVATE_LIBS@
Requires: liboxenc
Requires.private: libzmq libsodium
Cflags: -I${includedir}

1
oxen-encoding Submodule

@ -0,0 +1 @@
Subproject commit a0912ab4bf3b5e83b42715eff6f632c8912b21e4

38
oxenmq/address.cpp
View File

@ -5,9 +5,9 @@
#include <utility>
#include <stdexcept>
#include <ostream>
#include "hex.h"
#include "base32z.h"
#include "base64.h"
#include <oxenc/hex.h>
#include <oxenc/base32z.h>
#include <oxenc/base64.h>
namespace oxenmq {
@ -23,14 +23,14 @@ constexpr size_t enc_length(address::encoding enc) {
// given: for QR-friendly we only accept hex or base32z (since QR cannot handle base64's alphabet).
std::string decode_pubkey(std::string_view& in, bool qr) {
std::string pubkey;
if (in.size() >= 64 && is_hex(in.substr(0, 64))) {
pubkey = from_hex(in.substr(0, 64));
if (in.size() >= 64 && oxenc::is_hex(in.substr(0, 64))) {
pubkey = oxenc::from_hex(in.substr(0, 64));
in.remove_prefix(64);
} else if (in.size() >= 52 && is_base32z(in.substr(0, 52))) {
pubkey = from_base32z(in.substr(0, 52));
} else if (in.size() >= 52 && oxenc::is_base32z(in.substr(0, 52))) {
pubkey = oxenc::from_base32z(in.substr(0, 52));
in.remove_prefix(52);
} else if (!qr && in.size() >= 43 && is_base64(in.substr(0, 43))) {
pubkey = from_base64(in.substr(0, 43));
} else if (!qr && in.size() >= 43 && oxenc::is_base64(in.substr(0, 43))) {
pubkey = oxenc::from_base64(in.substr(0, 43));
in.remove_prefix(43);
if (!in.empty() && in.front() == '=')
in.remove_prefix(1); // allow (and eat) a padding byte at the end
@ -116,15 +116,15 @@ std::pair<std::string, std::string> parse_unix(std::string_view& addr, bool expe
std::pair<std::string, std::string> result;
if (expect_pubkey) {
size_t b64_len = addr.size() > 0 && addr.back() == '=' ? 44 : 43;
if (addr.size() > 64 && addr[addr.size() - 65] == '/' && is_hex(addr.substr(addr.size() - 64))) {
if (addr.size() > 64 && addr[addr.size() - 65] == '/' && oxenc::is_hex(addr.substr(addr.size() - 64))) {
result.first = std::string{addr.substr(0, addr.size() - 65)};
result.second = from_hex(addr.substr(addr.size() - 64));
} else if (addr.size() > 52 && addr[addr.size() - 53] == '/' && is_base32z(addr.substr(addr.size() - 52))) {
result.second = oxenc::from_hex(addr.substr(addr.size() - 64));
} else if (addr.size() > 52 && addr[addr.size() - 53] == '/' && oxenc::is_base32z(addr.substr(addr.size() - 52))) {
result.first = std::string{addr.substr(0, addr.size() - 53)};
result.second = from_base32z(addr.substr(addr.size() - 52));
} else if (addr.size() > b64_len && addr[addr.size() - b64_len - 1] == '/' && is_base64(addr.substr(addr.size() - b64_len))) {
result.second = oxenc::from_base32z(addr.substr(addr.size() - 52));
} else if (addr.size() > b64_len && addr[addr.size() - b64_len - 1] == '/' && oxenc::is_base64(addr.substr(addr.size() - b64_len))) {
result.first = std::string{addr.substr(0, addr.size() - b64_len - 1)};
result.second = from_base64(addr.substr(addr.size() - b64_len));
result.second = oxenc::from_base64(addr.substr(addr.size() - b64_len));
} else {
throw std::invalid_argument{"icp+curve:// requires a trailing /PUBKEY value, got: " + std::string{addr}};
}
@ -198,16 +198,16 @@ address& address::set_pubkey(std::string_view pk) {
std::string address::encode_pubkey(encoding enc) const {
std::string pk;
if (enc == encoding::hex)
pk = to_hex(pubkey);
pk = oxenc::to_hex(pubkey);
else if (enc == encoding::base32z)
pk = to_base32z(pubkey);
pk = oxenc::to_base32z(pubkey);
else if (enc == encoding::BASE32Z) {
pk = to_base32z(pubkey);
pk = oxenc::to_base32z(pubkey);
for (char& c : pk)
if (c >= 'a' && c <= 'z')
c = c - 'a' + 'A';
} else if (enc == encoding::base64) {
pk = to_base64(pubkey);
pk = oxenc::to_base64(pubkey);
if (pk.size() == 44 && pk.back() == '=')
pk.resize(43);
} else {

34
oxenmq/auth.cpp
View File

@ -1,5 +1,5 @@
#include "oxenmq.h"
#include "hex.h"
#include <oxenc/hex.h>
#include "oxenmq-internal.h"
#include <ostream>
#include <sstream>
@ -37,23 +37,23 @@ bool OxenMQ::proxy_check_auth(int64_t conn_id, bool outgoing, const peer_info& p
std::string reply;
if (!cat_call.first) {
OMQ_LOG(warn, "Invalid command '", command, "' sent by remote [", to_hex(peer.pubkey), "]/", peer_address(cmd));
OMQ_LOG(warn, "Invalid command '", command, "' sent by remote [", oxenc::to_hex(peer.pubkey), "]/", peer_address(cmd));
reply = "UNKNOWNCOMMAND";
} else if (peer.auth_level < cat_call.first->access.auth) {
OMQ_LOG(warn, "Access denied to ", command, " for peer [", to_hex(peer.pubkey), "]/", peer_address(cmd),
OMQ_LOG(warn, "Access denied to ", command, " for peer [", oxenc::to_hex(peer.pubkey), "]/", peer_address(cmd),
": peer auth level ", peer.auth_level, " < ", cat_call.first->access.auth);
reply = "FORBIDDEN";
} else if (cat_call.first->access.local_sn && !local_service_node) {
OMQ_LOG(warn, "Access denied to ", command, " for peer [", to_hex(peer.pubkey), "]/", peer_address(cmd),
OMQ_LOG(warn, "Access denied to ", command, " for peer [", oxenc::to_hex(peer.pubkey), "]/", peer_address(cmd),
": that command is only available when this OxenMQ is running in service node mode");
reply = "NOT_A_SERVICE_NODE";
} else if (cat_call.first->access.remote_sn && !peer.service_node) {
OMQ_LOG(warn, "Access denied to ", command, " for peer [", to_hex(peer.pubkey), "]/", peer_address(cmd),
OMQ_LOG(warn, "Access denied to ", command, " for peer [", oxenc::to_hex(peer.pubkey), "]/", peer_address(cmd),
": remote is not recognized as a service node");
reply = "FORBIDDEN_SN";
} else if (cat_call.second->second /*is_request*/ && data.empty()) {
OMQ_LOG(warn, "Received an invalid request for '", command, "' with no reply tag from remote [",
to_hex(peer.pubkey), "]/", peer_address(cmd));
oxenc::to_hex(peer.pubkey), "]/", peer_address(cmd));
reply = "NO_REPLY_TAG";
} else {
return true;
@ -75,7 +75,7 @@ bool OxenMQ::proxy_check_auth(int64_t conn_id, bool outgoing, const peer_info& p
send_message_parts(connections.at(conn_id), msgs);
} catch (const zmq::error_t& err) {
/* can't send: possibly already disconnected. Ignore. */
OMQ_LOG(debug, "Couldn't send auth failure message ", reply, " to peer [", to_hex(peer.pubkey), "]/", peer_address(cmd), ": ", err.what());
OMQ_LOG(debug, "Couldn't send auth failure message ", reply, " to peer [", oxenc::to_hex(peer.pubkey), "]/", peer_address(cmd), ": ", err.what());
}
return false;
@ -83,21 +83,21 @@ bool OxenMQ::proxy_check_auth(int64_t conn_id, bool outgoing, const peer_info& p
void OxenMQ::set_active_sns(pubkey_set pubkeys) {
if (proxy_thread.joinable()) {
auto data = bt_serialize(detail::serialize_object(std::move(pubkeys)));
auto data = oxenc::bt_serialize(detail::serialize_object(std::move(pubkeys)));
detail::send_control(get_control_socket(), "SET_SNS", data);
} else {
proxy_set_active_sns(std::move(pubkeys));
}
}
void OxenMQ::proxy_set_active_sns(std::string_view data) {
proxy_set_active_sns(detail::deserialize_object<pubkey_set>(bt_deserialize<uintptr_t>(data)));
proxy_set_active_sns(detail::deserialize_object<pubkey_set>(oxenc::bt_deserialize<uintptr_t>(data)));
}
void OxenMQ::proxy_set_active_sns(pubkey_set pubkeys) {
pubkey_set added, removed;
for (auto it = pubkeys.begin(); it != pubkeys.end(); ) {
auto& pk = *it;
if (pk.size() != 32) {
OMQ_LOG(warn, "Invalid private key of length ", pk.size(), " (", to_hex(pk), ") passed to set_active_sns");
OMQ_LOG(warn, "Invalid private key of length ", pk.size(), " (", oxenc::to_hex(pk), ") passed to set_active_sns");
it = pubkeys.erase(it);
continue;
}
@ -123,12 +123,12 @@ void OxenMQ::update_active_sns(pubkey_set added, pubkey_set removed) {
std::array<uintptr_t, 2> data;
data[0] = detail::serialize_object(std::move(added));
data[1] = detail::serialize_object(std::move(removed));
detail::send_control(get_control_socket(), "UPDATE_SNS", bt_serialize(data));
detail::send_control(get_control_socket(), "UPDATE_SNS", oxenc::bt_serialize(data));
} else {
proxy_update_active_sns(std::move(added), std::move(removed));
}
}
void OxenMQ::proxy_update_active_sns(bt_list_consumer data) {
void OxenMQ::proxy_update_active_sns(oxenc::bt_list_consumer data) {
auto added = detail::deserialize_object<pubkey_set>(data.consume_integer<uintptr_t>());
auto remed = detail::deserialize_object<pubkey_set>(data.consume_integer<uintptr_t>());
proxy_update_active_sns(std::move(added), std::move(remed));
@ -141,7 +141,7 @@ void OxenMQ::proxy_update_active_sns(pubkey_set added, pubkey_set removed) {
for (auto it = removed.begin(); it != removed.end(); ) {
const auto& pk = *it;
if (pk.size() != 32) {
OMQ_LOG(warn, "Invalid private key of length ", pk.size(), " (", to_hex(pk), ") passed to update_active_sns (removed)");
OMQ_LOG(warn, "Invalid private key of length ", pk.size(), " (", oxenc::to_hex(pk), ") passed to update_active_sns (removed)");
it = removed.erase(it);
} else if (!active_service_nodes.count(pk) || added.count(pk) /* added wins if in both */) {
it = removed.erase(it);
@ -153,7 +153,7 @@ void OxenMQ::proxy_update_active_sns(pubkey_set added, pubkey_set removed) {
for (auto it = added.begin(); it != added.end(); ) {
const auto& pk = *it;
if (pk.size() != 32) {
OMQ_LOG(warn, "Invalid private key of length ", pk.size(), " (", to_hex(pk), ") passed to update_active_sns (added)");
OMQ_LOG(warn, "Invalid private key of length ", pk.size(), " (", oxenc::to_hex(pk), ") passed to update_active_sns (added)");
it = added.erase(it);
} else if (active_service_nodes.count(pk)) {
it = added.erase(it);
@ -200,7 +200,7 @@ void OxenMQ::process_zap_requests() {
o << "\n[" << i << "]: ";
auto v = view(frames[i]);
if (i == 1 || i == 6)
o << to_hex(v);
o << oxenc::to_hex(v);
else
o << v;
}
@ -247,7 +247,7 @@ void OxenMQ::process_zap_requests() {
auto auth_domain = view(frames[2]);
size_t bind_id = (size_t) -1;
try {
bind_id = bt_deserialize<size_t>(view(frames[2]));
bind_id = oxenc::bt_deserialize<size_t>(view(frames[2]));
} catch (...) {}
if (bind_id >= bind.size()) {
@ -282,7 +282,7 @@ void OxenMQ::process_zap_requests() {
auto& user_id = response_vals[4];
if (bind[bind_id].curve) {
user_id.reserve(64);
to_hex(pubkey.begin(), pubkey.end(), std::back_inserter(user_id));
oxenc::to_hex(pubkey.begin(), pubkey.end(), std::back_inserter(user_id));
}
if (auth <= AuthLevel::denied || auth > AuthLevel::admin) {

280
oxenmq/base32z.h
View File

@ -27,277 +27,19 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <string>
#include <string_view>
#include <array>
#include <iterator>
#include <cassert>
#include "byte_type.h"
// Compatibility shim for oxenc includes
//
#include <oxenc/base32z.h>
namespace oxenmq {
namespace detail {
/// Compile-time generated lookup tables for base32z conversion. This is case insensitive (though
/// for byte -> b32z conversion we always produce lower case).
struct b32z_table {
// Store the 0-31 decoded value of every possible char; all the chars that aren't valid are set
// to 0. (If you don't trust your data, check it with is_base32z first, which uses these 0's
// to detect invalid characters -- which is why we want a full 256 element array).
char from_b32z_lut[256];
// Store the encoded character of every 0-31 (5 bit) value.
char to_b32z_lut[32];
// constexpr constructor that fills out the above (and should do it at compile time for any half
// decent compiler).
constexpr b32z_table() noexcept : from_b32z_lut{},
to_b32z_lut{
'y', 'b', 'n', 'd', 'r', 'f', 'g', '8', 'e', 'j', 'k', 'm', 'c', 'p', 'q', 'x',
'o', 't', '1', 'u', 'w', 'i', 's', 'z', 'a', '3', '4', '5', 'h', '7', '6', '9'
}
{
for (unsigned char c = 0; c < 32; c++) {
unsigned char x = to_b32z_lut[c];
from_b32z_lut[x] = c;
if (x >= 'a' && x <= 'z')
from_b32z_lut[x - 'a' + 'A'] = c;
}
}
// Convert a b32z encoded character into a 0-31 value
constexpr char from_b32z(unsigned char c) const noexcept { return from_b32z_lut[c]; }
// Convert a 0-31 value into a b32z encoded character
constexpr char to_b32z(unsigned char b) const noexcept { return to_b32z_lut[b]; }
} constexpr b32z_lut;
// This main point of this static assert is to force the compiler to compile-time build the constexpr tables.
static_assert(b32z_lut.from_b32z('w') == 20 && b32z_lut.from_b32z('T') == 17 && b32z_lut.to_b32z(5) == 'f', "");
} // namespace detail
/// Returns the number of characters required to encode a base32z string from the given number of bytes.
inline constexpr size_t to_base32z_size(size_t byte_size) { return (byte_size*8 + 4) / 5; } // ⌈bits/5⌉ because 5 bits per byte
/// Returns the (maximum) number of bytes required to decode a base32z string of the given size.
inline constexpr size_t from_base32z_size(size_t b32z_size) { return b32z_size*5 / 8; } // ⌊bits/8⌋
/// Iterable object for on-the-fly base32z encoding. Used internally, but also particularly useful
/// when converting from one encoding to another.
template <typename InputIt>
struct base32z_encoder final {
private:
InputIt _it, _end;
static_assert(sizeof(decltype(*_it)) == 1, "base32z_encoder requires chars/bytes input iterator");
// Number of bits held in r; will always be >= 5 until we are at the end.
int bits{_it != _end ? 8 : 0};
// Holds bits of data we've already read, which might belong to current or next chars
uint_fast16_t r{bits ? static_cast<unsigned char>(*_it) : (unsigned char)0};
public:
using iterator_category = std::input_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = char;
using reference = value_type;
using pointer = void;
base32z_encoder(InputIt begin, InputIt end) : _it{std::move(begin)}, _end{std::move(end)} {}
base32z_encoder end() { return {_end, _end}; }
bool operator==(const base32z_encoder& i) { return _it == i._it && bits == i.bits; }
bool operator!=(const base32z_encoder& i) { return !(*this == i); }
base32z_encoder& operator++() {
assert(bits >= 5);
// Discard the most significant 5 bits
bits -= 5;
r &= (1 << bits) - 1;
// If we end up with less than 5 significant bits then try to pull another 8 bits:
if (bits < 5 && _it != _end) {
if (++_it != _end) {
r = (r << 8) | static_cast<unsigned char>(*_it);
bits += 8;
} else if (bits > 0) {
// No more input bytes, so shift `r` to put the bits we have into the most
// significant bit position for the final character. E.g. if we have "11" we want
// the last character to be encoded "11000".
r <<= (5 - bits);
bits = 5;
}
}
return *this;
}
base32z_encoder operator++(int) { base32z_encoder copy{*this}; ++*this; return copy; }
char operator*() {
// Right-shift off the excess bits we aren't accessing yet
return detail::b32z_lut.to_b32z(r >> (bits - 5));
}
};
/// Converts bytes into a base32z encoded character sequence, writing them starting at `out`.
/// Returns the final value of out (i.e. the iterator positioned just after the last written base32z
/// character).
template <typename InputIt, typename OutputIt>
OutputIt to_base32z(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "to_base32z requires chars/bytes");
base32z_encoder it{begin, end};
return std::copy(it, it.end(), out);
}
/// Creates a base32z string from an iterator pair of a byte sequence.
template <typename It>
std::string to_base32z(It begin, It end) {
std::string base32z;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>) {
using std::distance;
base32z.reserve(to_base32z_size(distance(begin, end)));
}
to_base32z(begin, end, std::back_inserter(base32z));
return base32z;
}
/// Creates a base32z string from an iterable, std::string-like object
template <typename CharT>
std::string to_base32z(std::basic_string_view<CharT> s) { return to_base32z(s.begin(), s.end()); }
inline std::string to_base32z(std::string_view s) { return to_base32z<>(s); }
/// Returns true if the given [begin, end) range is an acceptable base32z string: specifically every
/// character must be in the base32z alphabet, and the string must be a valid encoding length that
/// could have been produced by to_base32z (i.e. some lengths are impossible).
template <typename It>
constexpr bool is_base32z(It begin, It end) {
static_assert(sizeof(decltype(*begin)) == 1, "is_base32z requires chars/bytes");
size_t count = 0;
constexpr bool random = std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>;
if constexpr (random) {
using std::distance;
count = distance(begin, end) % 8;
if (count == 1 || count == 3 || count == 6) // see below
return false;
}
for (; begin != end; ++begin) {
auto c = static_cast<unsigned char>(*begin);
if (detail::b32z_lut.from_b32z(c) == 0 && !(c == 'y' || c == 'Y'))
return false;
if constexpr (!random)
count++;
}
// Check for a valid length.
// - 5n + 0 bytes encodes to 8n chars (no padding bits)
// - 5n + 1 bytes encodes to 8n+2 chars (last 2 bits are padding)
// - 5n + 2 bytes encodes to 8n+4 chars (last 4 bits are padding)
// - 5n + 3 bytes encodes to 8n+5 chars (last 1 bit is padding)
// - 5n + 4 bytes encodes to 8n+7 chars (last 3 bits are padding)
if constexpr (!random)
if (count %= 8; count == 1 || count == 3 || count == 6)
return false;
return true;
}
/// Returns true if all elements in the string-like value are base32z characters
template <typename CharT>
constexpr bool is_base32z(std::basic_string_view<CharT> s) { return is_base32z(s.begin(), s.end()); }
constexpr bool is_base32z(std::string_view s) { return is_base32z<>(s); }
/// Iterable object for on-the-fly base32z decoding. Used internally, but also particularly useful
/// when converting from one encoding to another. The input range must be a valid base32z
/// encoded string.
///
/// Note that we ignore "padding" bits without requiring that they actually be 0. For instance, the
/// bytes "\ff\ff" are ideally encoded as "999o" (16 bits of 1s + 4 padding 0 bits), but we don't
/// require that the padding bits be 0. That is, "9999", "9993", etc. will all decode to the same
/// \ff\ff output string.
template <typename InputIt>
struct base32z_decoder final {
private:
InputIt _it, _end;
static_assert(sizeof(decltype(*_it)) == 1, "base32z_decoder requires chars/bytes input iterator");
uint_fast16_t in = 0;
int bits = 0; // number of bits loaded into `in`; will be in [8, 12] until we hit the end
public:
using iterator_category = std::input_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = char;
using reference = value_type;
using pointer = void;
base32z_decoder(InputIt begin, InputIt end) : _it{std::move(begin)}, _end{std::move(end)} {
if (_it != _end)
load_byte();
}
base32z_decoder end() { return {_end, _end}; }
bool operator==(const base32z_decoder& i) { return _it == i._it; }
bool operator!=(const base32z_decoder& i) { return _it != i._it; }
base32z_decoder& operator++() {
// Discard 8 most significant bits
bits -= 8;
in &= (1 << bits) - 1;
if (++_it != _end)
load_byte();
return *this;
}
base32z_decoder operator++(int) { base32z_decoder copy{*this}; ++*this; return copy; }
char operator*() {
return in >> (bits - 8);
}
private:
void load_in() {
in = in << 5
| detail::b32z_lut.from_b32z(static_cast<unsigned char>(*_it));
bits += 5;
}
void load_byte() {
load_in();
if (bits < 8 && ++_it != _end)
load_in();
// If we hit the _end iterator above then we hit the end of the input with fewer than 8 bits
// accumulated to make a full byte. For a properly encoded base32z string this should only
// be possible with 0-4 bits of all 0s; these are essentially "padding" bits (e.g. encoding
// 2 byte (16 bits) requires 4 b32z chars (20 bits), where only the first 16 bits are
// significant). Ideally any padding bits should be 0, but we don't check that and rather
// just ignore them.
//
// It also isn't possible to get here with 5-7 bits if the string passes `is_base32z`
// because the length checks we do there disallow such a length as valid. (If you were to
// pass such a string to us anyway then we are technically UB, but the current
// implementation just ignore the extra bits as if they are extra padding).
}
};
/// Converts a sequence of base32z digits to bytes. Undefined behaviour if any characters are not
/// valid base32z alphabet characters. It is permitted for the input and output ranges to overlap
/// as long as `out` is no later than `begin`.
///
template <typename InputIt, typename OutputIt>
OutputIt from_base32z(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "from_base32z requires chars/bytes");
base32z_decoder it{begin, end};
auto bend = it.end();
while (it != bend)
*out++ = static_cast<detail::byte_type_t<OutputIt>>(*it++);
return out;
}
/// Convert a base32z sequence into a std::string of bytes. Undefined behaviour if any characters
/// are not valid (case-insensitive) base32z characters.
template <typename It>
std::string from_base32z(It begin, It end) {
std::string bytes;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>) {
using std::distance;
bytes.reserve(from_base32z_size(distance(begin, end)));
}
from_base32z(begin, end, std::back_inserter(bytes));
return bytes;
}
/// Converts base32z digits from a std::string-like object into a std::string of bytes. Undefined
/// behaviour if any characters are not valid (case-insensitive) base32z characters.
template <typename CharT>
std::string from_base32z(std::basic_string_view<CharT> s) { return from_base32z(s.begin(), s.end()); }
inline std::string from_base32z(std::string_view s) { return from_base32z<>(s); }
using oxenc::to_base32z_size;
using oxenc::from_base32z_size;
using oxenc::base32z_encoder;
using oxenc::to_base32z;
using oxenc::is_base32z;
using oxenc::base32z_decoder;
using oxenc::from_base32z;
}

351
oxenmq/base64.h
View File

@ -27,347 +27,20 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <string>
#include <string_view>
#include <array>
#include <iterator>
#include <cassert>
#include "byte_type.h"
// Compatibility shim for oxenc includes
#include <oxenc/base64.h>
namespace oxenmq {
namespace detail {
/// Compile-time generated lookup tables for base64 conversion.
struct b64_table {
// Store the 0-63 decoded value of every possible char; all the chars that aren't valid are set
// to 0. (If you don't trust your data, check it with is_base64 first, which uses these 0's
// to detect invalid characters -- which is why we want a full 256 element array).
char from_b64_lut[256];
// Store the encoded character of every 0-63 (6 bit) value.
char to_b64_lut[64];
// constexpr constructor that fills out the above (and should do it at compile time for any half
// decent compiler).
constexpr b64_table() noexcept : from_b64_lut{}, to_b64_lut{} {
for (unsigned char c = 0; c < 26; c++) {
from_b64_lut[(unsigned char)('A' + c)] = 0 + c;
to_b64_lut[ (unsigned char)( 0 + c)] = 'A' + c;
}
for (unsigned char c = 0; c < 26; c++) {
from_b64_lut[(unsigned char)('a' + c)] = 26 + c;
to_b64_lut[ (unsigned char)(26 + c)] = 'a' + c;
}
for (unsigned char c = 0; c < 10; c++) {
from_b64_lut[(unsigned char)('0' + c)] = 52 + c;
to_b64_lut[ (unsigned char)(52 + c)] = '0' + c;
}
to_b64_lut[62] = '+'; from_b64_lut[(unsigned char) '+'] = 62;
to_b64_lut[63] = '/'; from_b64_lut[(unsigned char) '/'] = 63;
}
// Convert a b64 encoded character into a 0-63 value
constexpr char from_b64(unsigned char c) const noexcept { return from_b64_lut[c]; }
// Convert a 0-31 value into a b64 encoded character
constexpr char to_b64(unsigned char b) const noexcept { return to_b64_lut[b]; }
} constexpr b64_lut;
// This main point of this static assert is to force the compiler to compile-time build the constexpr tables.
static_assert(b64_lut.from_b64('/') == 63 && b64_lut.from_b64('7') == 59 && b64_lut.to_b64(38) == 'm', "");
} // namespace detail
/// Returns the number of characters required to encode a base64 string from the given number of bytes.
inline constexpr size_t to_base64_size(size_t byte_size, bool padded = true) {
return padded
? (byte_size + 2) / 3 * 4 // bytes*4/3, rounded up to the next multiple of 4
: (byte_size * 4 + 2) / 3; // ⌈bytes*4/3⌉
}
/// Returns the (maximum) number of bytes required to decode a base64 string of the given size.
/// Note that this may overallocate by 1-2 bytes if the size includes 1-2 padding chars.
inline constexpr size_t from_base64_size(size_t b64_size) {
return b64_size * 3 / 4; // == ⌊bits/8⌋; floor because we ignore trailing "impossible" bits (see below)
}
/// Iterable object for on-the-fly base64 encoding. Used internally, but also particularly useful
/// when converting from one encoding to another.
template <typename InputIt>
struct base64_encoder final {
private:
InputIt _it, _end;
static_assert(sizeof(decltype(*_it)) == 1, "base64_encoder requires chars/bytes input iterator");
// How much padding (at most) we can add at the end
int padding;
// Number of bits held in r; will always be >= 6 until we are at the end.
int bits{_it != _end ? 8 : 0};
// Holds bits of data we've already read, which might belong to current or next chars
uint_fast16_t r{bits ? static_cast<unsigned char>(*_it) : (unsigned char)0};
public:
using iterator_category = std::input_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = char;
using reference = value_type;
using pointer = void;
base64_encoder(InputIt begin, InputIt end, bool padded = true)
: _it{std::move(begin)}, _end{std::move(end)}, padding{padded} {}
base64_encoder end() { return {_end, _end, false}; }
bool operator==(const base64_encoder& i) { return _it == i._it && bits == i.bits && padding == i.padding; }
bool operator!=(const base64_encoder& i) { return !(*this == i); }
base64_encoder& operator++() {
if (bits == 0) {
padding--;
return *this;
}
assert(bits >= 6);
// Discard the most significant 6 bits
bits -= 6;
r &= (1 << bits) - 1;
// If we end up with less than 6 significant bits then try to pull another 8 bits:
if (bits < 6 && _it != _end) {
if (++_it != _end) {
r = (r << 8) | static_cast<unsigned char>(*_it);
bits += 8;
} else if (bits > 0) {
// No more input bytes, so shift `r` to put the bits we have into the most
// significant bit position for the final character, and figure out how many padding
// bytes we want to append. E.g. if we have "11" we want
// the last character to be encoded "110000".
if (padding) {
// padding should be:
// 3n+0 input => 4n output, no padding, handled below
// 3n+1 input => 4n+2 output + 2 padding; we'll land here with 2 trailing bits
// 3n+2 input => 4n+3 output + 1 padding; we'll land here with 4 trailing bits
padding = 3 - bits / 2;
}
r <<= (6 - bits);
bits = 6;
} else {
padding = 0; // No excess bits, so input was a multiple of 3 and thus no padding
}
}
return *this;
}
base64_encoder operator++(int) { base64_encoder copy{*this}; ++*this; return copy; }
char operator*() {
if (bits == 0 && padding)
return '=';
// Right-shift off the excess bits we aren't accessing yet
return detail::b64_lut.to_b64(r >> (bits - 6));
}
};
/// Converts bytes into a base64 encoded character sequence, writing them starting at `out`.
/// Returns the final value of out (i.e. the iterator positioned just after the last written base64
/// character).
template <typename InputIt, typename OutputIt>
OutputIt to_base64(InputIt begin, InputIt end, OutputIt out, bool padded = true) {
static_assert(sizeof(decltype(*begin)) == 1, "to_base64 requires chars/bytes");
auto it = base64_encoder{begin, end, padded};
return std::copy(it, it.end(), out);
}
/// Creates and returns a base64 string from an iterator pair of a character sequence. The
/// resulting string will have '=' padding, if appropriate.
template <typename It>
std::string to_base64(It begin, It end) {
std::string base64;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>) {
using std::distance;
base64.reserve(to_base64_size(distance(begin, end)));
}
to_base64(begin, end, std::back_inserter(base64));
return base64;
}
/// Creates and returns a base64 string from an iterator pair of a character sequence. The
/// resulting string will not be padded.
template <typename It>
std::string to_base64_unpadded(It begin, It end) {
std::string base64;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>) {
using std::distance;
base64.reserve(to_base64_size(distance(begin, end), false));
}
to_base64(begin, end, std::back_inserter(base64), false);
return base64;
}
/// Creates a base64 string from an iterable, std::string-like object. The string will have '='
/// padding, if appropriate.
template <typename CharT>
std::string to_base64(std::basic_string_view<CharT> s) { return to_base64(s.begin(), s.end()); }
inline std::string to_base64(std::string_view s) { return to_base64<>(s); }
/// Creates a base64 string from an iterable, std::string-like object. The string will not be
/// padded.
template <typename CharT>
std::string to_base64_unpadded(std::basic_string_view<CharT> s) { return to_base64_unpadded(s.begin(), s.end()); }
inline std::string to_base64_unpadded(std::string_view s) { return to_base64_unpadded<>(s); }
/// Returns true if the range is a base64 encoded value; we allow (but do not require) '=' padding,
/// but only at the end, only 1 or 2, and only if it pads out the total to a multiple of 4.
/// Otherwise the string must contain only valid base64 characters, and must not have a length of
/// 4n+1 (because that cannot be produced by base64 encoding).
template <typename It>
constexpr bool is_base64(It begin, It end) {
static_assert(sizeof(decltype(*begin)) == 1, "is_base64 requires chars/bytes");
using std::distance;
using std::prev;
size_t count = 0;
constexpr bool random = std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>;
if constexpr (random) {
count = distance(begin, end) % 4;
if (count == 1)
return false;
}
// Allow 1 or 2 padding chars *if* they pad it to a multiple of 4.
if (begin != end && distance(begin, end) % 4 == 0) {
auto last = prev(end);
if (static_cast<unsigned char>(*last) == '=')
end = last--;
if (static_cast<unsigned char>(*last) == '=')
end = last;
}
for (; begin != end; ++begin) {
auto c = static_cast<unsigned char>(*begin);
if (detail::b64_lut.from_b64(c) == 0 && c != 'A')
return false;
if constexpr (!random)
count++;
}
if constexpr (!random)
if (count % 4 == 1) // base64 encoding will produce 4n, 4n+2, 4n+3, but never 4n+1
return false;
return true;
}
/// Returns true if the string-like value is a base64 encoded value
template <typename CharT>
constexpr bool is_base64(std::basic_string_view<CharT> s) { return is_base64(s.begin(), s.end()); }
constexpr bool is_base64(std::string_view s) { return is_base64(s.begin(), s.end()); }
/// Iterable object for on-the-fly base64 decoding. Used internally, but also particularly useful
/// when converting from one encoding to another. The input range must be a valid base64 encoded
/// string (with or without padding).
///
/// Note that we ignore "padding" bits without requiring that they actually be 0. For instance, the
/// bytes "\ff\ff" are ideally encoded as "//8=" (16 bits of 1s + 2 padding 0 bits, then a full
/// 6-bit padding char). We don't, however, require that the padding bits be 0. That is, "///=",
/// "//9=", "//+=", etc. will all decode to the same \ff\ff output string.
template <typename InputIt>
struct base64_decoder final {
private:
InputIt _it, _end;
static_assert(sizeof(decltype(*_it)) == 1, "base64_decoder requires chars/bytes input iterator");
uint_fast16_t in = 0;
int bits = 0; // number of bits loaded into `in`; will be in [8, 12] until we hit the end
public:
using iterator_category = std::input_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = char;
using reference = value_type;
using pointer = void;
base64_decoder(InputIt begin, InputIt end) : _it{std::move(begin)}, _end{std::move(end)} {
if (_it != _end)
load_byte();
}
base64_decoder end() { return {_end, _end}; }
bool operator==(const base64_decoder& i) { return _it == i._it; }
bool operator!=(const base64_decoder& i) { return _it != i._it; }
base64_decoder& operator++() {
// Discard 8 most significant bits
bits -= 8;
in &= (1 << bits) - 1;
if (++_it != _end)
load_byte();
return *this;
}
base64_decoder operator++(int) { base64_decoder copy{*this}; ++*this; return copy; }
char operator*() {
return in >> (bits - 8);
}
private:
void load_in() {
// We hit padding trying to read enough for a full byte, so we're done. (And since you were
// already supposed to have checked validity with is_base64, the padding can only be at the
// end).
auto c = static_cast<unsigned char>(*_it);
if (c == '=') {
_it = _end;
bits = 0;
return;
}
in = in << 6
| detail::b64_lut.from_b64(c);
bits += 6;
}
void load_byte() {
load_in();
if (bits && bits < 8 && ++_it != _end)
load_in();
// If we hit the _end iterator above then we hit the end of the input (or hit padding) with
// fewer than 8 bits accumulated to make a full byte. For a properly encoded base64 string
// this should only be possible with 0, 2, or 4 bits of all 0s; these are essentially
// "padding" bits (e.g. encoding 2 byte (16 bits) requires 3 b64 chars (18 bits), where
// only the first 16 bits are significant). Ideally any padding bits should be 0, but we
// don't check that and rather just ignore them.
}
};
/// Converts a sequence of base64 digits to bytes. Undefined behaviour if any characters are not
/// valid base64 alphabet characters. It is permitted for the input and output ranges to overlap as
/// long as `out` is no later than `begin`. Trailing padding characters are permitted but not
/// required. Returns the final value of out (that is, the iterator positioned just after the
/// last written character).
///
/// It is possible to provide "impossible" base64 encoded values; for example "YWJja" which has 30
/// bits of data even though a base64 encoded byte string should have 24 (4 chars) or 36 (6 chars)
/// bits for a 3- and 4-byte input, respectively. We ignore any such "impossible" bits, and
/// similarly ignore impossible bits in the bit "overhang"; that means "YWJjZA==" (the proper
/// encoding of "abcd") and "YWJjZB", "YWJjZC", ..., "YWJjZP" all decode to the same "abcd" value:
/// the last 4 bits of the last character are essentially considered padding.
template <typename InputIt, typename OutputIt>
OutputIt from_base64(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "from_base64 requires chars/bytes");
base64_decoder it{begin, end};
auto bend = it.end();
while (it != bend)
*out++ = static_cast<detail::byte_type_t<OutputIt>>(*it++);
return out;
}
/// Converts base64 digits from a iterator pair of characters into a std::string of bytes.
/// Undefined behaviour if any characters are not valid base64 characters.
template <typename It>
std::string from_base64(It begin, It end) {
std::string bytes;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>) {
using std::distance;
bytes.reserve(from_base64_size(distance(begin, end)));
}
from_base64(begin, end, std::back_inserter(bytes));
return bytes;
}
/// Converts base64 digits from a std::string-like object into a std::string of bytes. Undefined
/// behaviour if any characters are not valid base64 characters.
template <typename CharT>
std::string from_base64(std::basic_string_view<CharT> s) { return from_base64(s.begin(), s.end()); }
inline std::string from_base64(std::string_view s) { return from_base64<>(s); }
using oxenc::to_base64_size;
using oxenc::from_base64_size;
using oxenc::base64_encoder;
using oxenc::to_base64;
using oxenc::to_base64_unpadded;
using oxenc::is_base64;
using oxenc::base64_decoder;
using oxenc::from_base64;
}

2
oxenmq/batch.h
View File

@ -273,7 +273,7 @@ void OxenMQ::batch(Batch<R>&& batch) {
throw std::logic_error("Cannot batch a a job batch with 0 jobs");
// Need to send this over to the proxy thread via the base class pointer. It assumes ownership.
auto* baseptr = static_cast<detail::Batch*>(new Batch<R>(std::move(batch)));
detail::send_control(get_control_socket(), "BATCH", bt_serialize(reinterpret_cast<uintptr_t>(baseptr)));
detail::send_control(get_control_socket(), "BATCH", oxenc::bt_serialize(reinterpret_cast<uintptr_t>(baseptr)));
}
}

304
oxenmq/bt_producer.h
View File

@ -1,306 +1,12 @@
#pragma once
#include <cassert>
#include <charconv>
#include <stdexcept>
#include <string_view>
#include <unordered_map>
#include <utility>
#include <variant>
#include <oxenc/bt_producer.h>
// Compatibility shim for oxenc includes
namespace oxenmq {
using namespace std::literals;
class bt_dict_producer;
#if defined(__APPLE__) && defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500
#define OXENMQ_APPLE_TO_CHARS_WORKAROUND
/// Really simplistic version of std::to_chars on Apple, because Apple doesn't allow `std::to_chars`
/// to be used if targetting anything before macOS 10.15. The buffer must have at least 20 chars of
/// space (for int types up to 64-bit); we return a pointer one past the last char written.
template <typename IntType>
char* apple_to_chars10(char* buf, IntType val) {
static_assert(std::is_integral_v<IntType> && sizeof(IntType) <= 64);
if constexpr (std::is_signed_v<IntType>) {
if (val < 0) {
buf[0] = '-';
return apple_to_chars10(buf+1, static_cast<std::make_unsigned_t<IntType>>(-val));
}
}
// write it to the buffer in reverse (because we don't know how many chars we'll need yet, but
// writing in reverse will figure that out).
char* pos = buf;
do {
*pos++ = '0' + static_cast<char>(val % 10);
val /= 10;
} while (val > 0);
// Reverse the digits into the right order
int swaps = (pos - buf) / 2;
for (int i = 0; i < swaps; i++)
std::swap(buf[i], pos[-1 - i]);
return pos;
}
#endif
/// Class that allows you to build a bt-encoded list manually, without copying or allocating memory.
/// This is essentially the reverse of bt_list_consumer: where it lets you stream-parse a buffer,
/// this class lets you build directly into a buffer that you own.
///
/// Out-of-buffer-space errors throw
class bt_list_producer {
friend class bt_dict_producer;
// Our pointers to the next write position and the past-the-end pointer of the buffer.
using buf_span = std::pair<char*, char*>;
// Our data is a begin/end pointer pair for the root list, or a pointer to our parent if a
// sublist.
std::variant<buf_span, bt_list_producer*, bt_dict_producer*> data;
// Reference to the write buffer; this is simply a reference to the value inside `data` for the
// root element, and a pointer to the root's value for sublists/subdicts.
buf_span& buffer;
// True indicates we have an open child list/dict
bool has_child = false;
// The range that contains this currently serialized value; `from` equals wherever the `l` was
// written that started this list and `to` is one past the `e` that ends it. Note that `to`
// will always be ahead of `buf_span.first` because we always write the `e`s to close open lists
// but these `e`s don't advance the write position (they will be overwritten if we append).
const char* const from;
const char* to;
// Sublist constructors
bt_list_producer(bt_list_producer* parent, std::string_view prefix = "l"sv);
bt_list_producer(bt_dict_producer* parent, std::string_view prefix = "l"sv);
// Does the actual appending to the buffer, and throwing if we'd overrun. If advance is false
// then we append without moving the buffer pointer (primarily when we append intermediate `e`s
// that we will overwrite if more data is added). This means that the next write will overwrite
// whatever was previously written by an `advance=false` call.
void buffer_append(std::string_view d, bool advance = true);
// Appends the 'e's into the buffer to close off open sublists/dicts *without* advancing the
// buffer position; we do this after each append so that the buffer always contains valid
// encoded data, even while we are still appending to it, and so that appending something raises
// a length_error if appending it would not leave enough space for the required e's to close the
// open list(s)/dict(s).
void append_intermediate_ends(size_t count = 1);
// Writes an integer to the given buffer; returns the one-past-the-data pointer. Up to 20 bytes
// will be written and must be available in buf. Used for both string and integer
// serialization.
template <typename IntType>
char* write_integer(IntType val, char* buf) {
static_assert(sizeof(IntType) <= 64);
#ifndef OXENMQ_APPLE_TO_CHARS_WORKAROUND
auto [ptr, ec] = std::to_chars(buf, buf+20, val);
assert(ec == std::errc());
return ptr;
#else
// Hate apple.
return apple_to_chars10(buf, val);
#endif
}
// Serializes an integer value and appends it to the output buffer. Does not call
// append_intermediate_ends().
template <typename IntType, std::enable_if_t<std::is_integral_v<IntType>, int> = 0>
void append_impl(IntType val) {
char buf[22]; // 'i' + base10 representation + 'e'
buf[0] = 'i';
auto* ptr = write_integer(val, buf+1);
*ptr++ = 'e';
buffer_append({buf, static_cast<size_t>(ptr-buf)});
}
// Appends a string value, but does not call append_intermediate_ends()
void append_impl(std::string_view s);
public:
bt_list_producer() = delete;
bt_list_producer(const bt_list_producer&) = delete;
bt_list_producer& operator=(const bt_list_producer&) = delete;
bt_list_producer& operator=(bt_list_producer&&) = delete;
bt_list_producer(bt_list_producer&& other);
~bt_list_producer();
/// Constructs a list producer that writes into the range [begin, end). If a write would go
/// beyond the end of the buffer an exception is raised. Note that this will happen during
/// construction if the given buffer is not large enough to contain the `le` encoding of an
/// empty list.
bt_list_producer(char* begin, char* end);
/// Constructs a list producer that writes into the range [begin, begin+size). If a write would
/// go beyond the end of the buffer an exception is raised.
bt_list_producer(char* begin, size_t len) : bt_list_producer{begin, begin + len} {}
/// Returns a string_view into the currently serialized data buffer. Note that the returned
/// view includes the `e` list end serialization markers which will be overwritten if the list
/// (or an active sublist/subdict) is appended to.
std::string_view view() const {
return {from, static_cast<size_t>(to-from)};
}
/// Returns the end position in the buffer.
const char* end() const { return to; }
/// Appends an element containing binary string data
void append(std::string_view data);
bt_list_producer& operator+=(std::string_view data) { append(data); return *this; }
/// Appends an integer
template <typename IntType, std::enable_if_t<std::is_integral_v<IntType>, int> = 0>
void append(IntType i) {
if (has_child) throw std::logic_error{"Cannot append to list when a sublist is active"};
append_impl(i);
append_intermediate_ends();
}
template <typename IntType, std::enable_if_t<std::is_integral_v<IntType>, int> = 0>
bt_list_producer& operator+=(IntType i) { append(i); return *this; }
/// Appends elements from the range [from, to) to the list. This does *not* append the elements
/// as a sublist: for that you should use something like: `l.append_list().append(from, to);`
template <typename ForwardIt>
void append(ForwardIt from, ForwardIt to) {
if (has_child) throw std::logic_error{"Cannot append to list when a sublist is active"};
while (from != to)
append_impl(*from++);
append_intermediate_ends();
}
/// Appends a sublist to this list. Returns a new bt_list_producer that references the parent
/// list. The parent cannot be added to until the sublist is destroyed. This is meant to be
/// used via RAII:
///
/// buf data[16];
/// bt_list_producer list{data, sizeof(data)};
/// {
/// auto sublist = list.append_list();
/// sublist.append(42);
/// }
/// list.append(1);
/// // `data` now contains: `lli42eei1ee`
///
/// If doing more complex lifetime management, take care not to allow the child instance to
/// outlive the parent.
bt_list_producer append_list();
/// Appends a dict to this list. Returns a new bt_dict_producer that references the parent
/// list. The parent cannot be added to until the subdict is destroyed. This is meant to be
/// used via RAII (see append_list() for details).
///
/// If doing more complex lifetime management, take care not to allow the child instance to
/// outlive the parent.
bt_dict_producer append_dict();
};
/// Class that allows you to build a bt-encoded dict manually, without copying or allocating memory.
/// This is essentially the reverse of bt_dict_consumer: where it lets you stream-parse a buffer,
/// this class lets you build directly into a buffer that you own.
///
/// Note that bt-encoded dicts *must* be produced in (ASCII) ascending key order, but that this is
/// only tracked/enforced for non-release builds (i.e. without -DNDEBUG).
class bt_dict_producer : bt_list_producer {
friend class bt_list_producer;
// Subdict constructors
bt_dict_producer(bt_list_producer* parent);
bt_dict_producer(bt_dict_producer* parent);
// Checks a just-written key string to make sure it is monotonically increasing from the last
// key. Does nothing in a release build.
#ifdef NDEBUG
constexpr void check_incrementing_key(size_t) const {}
#else
// String view into the buffer where we wrote the previous key.
std::string_view last_key;
void check_incrementing_key(size_t size);
#endif
public:
// Construction is identical to bt_list_producer
using bt_list_producer::bt_list_producer;
/// Returns a string_view into the currently serialized data buffer. Note that the returned
/// view includes the `e` dict end serialization markers which will be overwritten if the dict
/// (or an active sublist/subdict) is appended to.
std::string_view view() const { return bt_list_producer::view(); }
/// Appends a key-value pair with a string or integer value. The key must be > the last key
/// added, but this is only enforced (with an assertion) in debug builds.
template <typename T, std::enable_if_t<std::is_convertible_v<T, std::string_view> || std::is_integral_v<T>, int> = 0>
void append(std::string_view key, const T& value) {
if (has_child) throw std::logic_error{"Cannot append to list when a sublist is active"};
append_impl(key);
check_incrementing_key(key.size());
append_impl(value);
append_intermediate_ends();
}
/// Appends pairs from the range [from, to) to the dict. Elements must have a .first
/// convertible to a string_view, and a .second that is either string view convertible or an
/// integer. This does *not* append the elements as a subdict: for that you should use
/// something like: `l.append_dict().append(key, from, to);`
///
/// Also note that the range *must* be sorted by keys, which means either using an ordered
/// container (e.g. std::map) or a manually ordered container (such as a vector or list of
/// pairs). unordered_map, however, is not acceptable.
template <typename ForwardIt, std::enable_if_t<!std::is_convertible_v<ForwardIt, std::string_view>, int> = 0>
void append(ForwardIt from, ForwardIt to) {
if (has_child) throw std::logic_error{"Cannot append to list when a sublist is active"};
using KeyType = std::remove_cv_t<std::decay_t<decltype(from->first)>>;
using ValType = std::decay_t<decltype(from->second)>;
static_assert(std::is_convertible_v<decltype(from->first), std::string_view>);
static_assert(std::is_convertible_v<ValType, std::string_view> || std::is_integral_v<ValType>);
using BadUnorderedMap = std::unordered_map<KeyType, ValType>;
static_assert(!( // Disallow unordered_map iterators because they are not going to be ordered.
std::is_same_v<typename BadUnorderedMap::iterator, ForwardIt> ||
std::is_same_v<typename BadUnorderedMap::const_iterator, ForwardIt>));
while (from != to) {
const auto& [k, v] = *from++;
append_impl(k);
check_incrementing_key(k.size());
append_impl(v);
}
append_intermediate_ends();
}
/// Appends a sub-dict value to this dict with the given key. Returns a new bt_dict_producer
/// that references the parent dict. The parent cannot be added to until the subdict is
/// destroyed. Key must be (ascii-comparison) larger than the previous key.
///
/// This is meant to be used via RAII:
///
/// buf data[32];
/// bt_dict_producer dict{data, sizeof(data)};
/// {
/// auto subdict = dict.begin_dict("myKey");
/// subdict.append("x", 42);
/// }
/// dict.append("y", "");
/// // `data` now contains: `d5:myKeyd1:xi42ee1:y0:e`
///
/// If doing more complex lifetime management, take care not to allow the child instance to
/// outlive the parent.
bt_dict_producer append_dict(std::string_view key);
/// Appends a list to this dict with the given key (which must be ascii-larger than the previous
/// key). Returns a new bt_list_producer that references the parent dict. The parent cannot be
/// added to until the sublist is destroyed.
///
/// This is meant to be used via RAII (see append_dict() for details).
///
/// If doing more complex lifetime management, take care not to allow the child instance to
/// outlive the parent.
bt_list_producer append_list(std::string_view key);
};
using oxenc::bt_list_producer;
using oxenc::bt_dict_producer;
} // namespace oxenmq

369
oxenmq/bt_serialize.cpp
View File

@ -1,369 +0,0 @@
// Copyright (c) 2019-2021, The Oxen Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "bt_serialize.h"
#include "bt_producer.h"
#include "variant.h"
#include <cassert>
#include <iterator>
namespace oxenmq {
namespace detail {
/// Reads digits into an unsigned 64-bit int.
uint64_t extract_unsigned(std::string_view& s) {
if (s.empty())
throw bt_deserialize_invalid{"Expected 0-9 but found end of string"};
if (s[0] < '0' || s[0] > '9')
throw bt_deserialize_invalid("Expected 0-9 but found '"s + s[0]);
uint64_t uval = 0;
while (!s.empty() && (s[0] >= '0' && s[0] <= '9')) {
uint64_t bigger = uval * 10 + (s[0] - '0');
s.remove_prefix(1);
if (bigger < uval) // overflow
throw bt_deserialize_invalid("Integer deserialization failed: value is too large for a 64-bit int");
uval = bigger;
}
return uval;
}
void bt_deserialize<std::string_view>::operator()(std::string_view& s, std::string_view& val) {
if (s.size() < 2) throw bt_deserialize_invalid{"Deserialize failed: given data is not an bt-encoded string"};
if (s[0] < '0' || s[0] > '9')
throw bt_deserialize_invalid_type{"Expected 0-9 but found '"s + s[0] + "'"};
auto len = static_cast<size_t>(extract_unsigned(s));
if (s.empty() || s[0] != ':')
throw bt_deserialize_invalid{"Did not find expected ':' during string deserialization"};
s.remove_prefix(1);
if (len > s.size())
throw bt_deserialize_invalid{"String deserialization failed: encoded string length is longer than the serialized data"};
val = {s.data(), len};
s.remove_prefix(len);
}
// Check that we are on a 2's complement architecture. It's highly unlikely that this code ever
// runs on a non-2s-complement architecture (especially since C++20 requires a two's complement
// signed value behaviour), but check at compile time anyway because we rely on these relations
// below.
static_assert(std::numeric_limits<int64_t>::min() + std::numeric_limits<int64_t>::max() == -1 &&
static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) + uint64_t{1} == (uint64_t{1} << 63),
"Non 2s-complement architecture not supported!");
std::pair<uint64_t, bool> bt_deserialize_integer(std::string_view& s) {
// Smallest possible encoded integer is 3 chars: "i0e"
if (s.size() < 3) throw bt_deserialize_invalid("Deserialization failed: end of string found where integer expected");
if (s[0] != 'i') throw bt_deserialize_invalid_type("Deserialization failed: expected 'i', found '"s + s[0] + '\'');
s.remove_prefix(1);
std::pair<uint64_t, bool> result;
if (s[0] == '-') {
result.second = true;
s.remove_prefix(1);
}
result.first = extract_unsigned(s);
if (s.empty())
throw bt_deserialize_invalid("Integer deserialization failed: encountered end of string before integer was finished");
if (s[0] != 'e')
throw bt_deserialize_invalid("Integer deserialization failed: expected digit or 'e', found '"s + s[0] + '\'');
s.remove_prefix(1);
if (result.second /*negative*/ && result.first > (uint64_t{1} << 63))
throw bt_deserialize_invalid("Deserialization of integer failed: negative integer value is too large for a 64-bit signed int");
return result;
}
template struct bt_deserialize<int64_t>;
template struct bt_deserialize<uint64_t>;
void bt_deserialize<bt_value, void>::operator()(std::string_view& s, bt_value& val) {
if (s.size() < 2) throw bt_deserialize_invalid("Deserialization failed: end of string found where bt-encoded value expected");
switch (s[0]) {
case 'd': {
bt_dict dict;
bt_deserialize<bt_dict>{}(s, dict);
val = std::move(dict);
break;
}
case 'l': {
bt_list list;
bt_deserialize<bt_list>{}(s, list);
val = std::move(list);
break;
}
case 'i': {
auto [magnitude, negative] = bt_deserialize_integer(s);
if (negative) val = -static_cast<int64_t>(magnitude);
else val = magnitude;
break;
}
case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': {
std::string str;
bt_deserialize<std::string>{}(s, str);
val = std::move(str);
break;
}
default:
throw bt_deserialize_invalid("Deserialize failed: encountered invalid value '"s + s[0] + "'; expected one of [0-9idl]");
}
}
} // namespace detail
bt_list_consumer::bt_list_consumer(std::string_view data_) : data{std::move(data_)} {
if (data.empty()) throw std::runtime_error{"Cannot create a bt_list_consumer with an empty string_view"};
if (data[0] != 'l') throw std::runtime_error{"Cannot create a bt_list_consumer with non-list data"};
data.remove_prefix(1);
}
/// Attempt to parse the next value as a string (and advance just past it). Throws if the next
/// value is not a string.
std::string_view bt_list_consumer::consume_string_view() {
if (data.empty())
throw bt_deserialize_invalid{"expected a string, but reached end of data"};
else if (!is_string())
throw bt_deserialize_invalid_type{"expected a string, but found "s + data.front()};
std::string_view next{data}, result;
detail::bt_deserialize<std::string_view>{}(next, result);
data = next;
return result;
}
std::string bt_list_consumer::consume_string() {
return std::string{consume_string_view()};
}
/// Consumes a value without returning it.
void bt_list_consumer::skip_value() {
if (is_string())
consume_string_view();
else if (is_integer())
detail::bt_deserialize_integer(data);
else if (is_list())
consume_list_data();
else if (is_dict())
consume_dict_data();
else
throw bt_deserialize_invalid_type{"next bt value has unknown type"};
}
std::string_view bt_list_consumer::consume_list_data() {
auto start = data.begin();
if (data.size() < 2 || !is_list()) throw bt_deserialize_invalid_type{"next bt value is not a list"};
data.remove_prefix(1); // Descend into the sublist, consume the "l"
while (!is_finished()) {
skip_value();
if (data.empty())
throw bt_deserialize_invalid{"bt list consumption failed: hit the end of string before the list was done"};
}
data.remove_prefix(1); // Back out from the sublist, consume the "e"
return {start, static_cast<size_t>(std::distance(start, data.begin()))};
}
std::string_view bt_list_consumer::consume_dict_data() {
auto start = data.begin();
if (data.size() < 2 || !is_dict()) throw bt_deserialize_invalid_type{"next bt value is not a dict"};
data.remove_prefix(1); // Descent into the dict, consumer the "d"
while (!is_finished()) {
consume_string_view(); // Key is always a string
if (!data.empty())
skip_value();
if (data.empty())
throw bt_deserialize_invalid{"bt dict consumption failed: hit the end of string before the dict was done"};
}
data.remove_prefix(1); // Back out of the dict, consume the "e"
return {start, static_cast<size_t>(std::distance(start, data.begin()))};
}
bt_dict_consumer::bt_dict_consumer(std::string_view data_) {
data = std::move(data_);
if (data.empty()) throw std::runtime_error{"Cannot create a bt_dict_consumer with an empty string_view"};
if (data.size() < 2 || data[0] != 'd') throw std::runtime_error{"Cannot create a bt_dict_consumer with non-dict data"};
data.remove_prefix(1);
}
bool bt_dict_consumer::consume_key() {
if (key_.data())
return true;
if (data.empty()) throw bt_deserialize_invalid_type{"expected a key or dict end, found end of string"};
if (data[0] == 'e') return false;
key_ = bt_list_consumer::consume_string_view();
if (data.empty() || data[0] == 'e')
throw bt_deserialize_invalid{"dict key isn't followed by a value"};
return true;
}
std::pair<std::string_view, std::string_view> bt_dict_consumer::next_string() {
if (!is_string())
throw bt_deserialize_invalid_type{"expected a string, but found "s + data.front()};
std::pair<std::string_view, std::string_view> ret;
ret.second = bt_list_consumer::consume_string_view();
ret.first = flush_key();
return ret;
}
bt_list_producer::bt_list_producer(bt_list_producer* parent, std::string_view prefix)
: data{parent}, buffer{parent->buffer}, from{buffer.first} {
parent->has_child = true;
buffer_append(prefix);
append_intermediate_ends();
}
bt_list_producer::bt_list_producer(bt_dict_producer* parent, std::string_view prefix)
: data{parent}, buffer{parent->buffer}, from{buffer.first} {
parent->has_child = true;
buffer_append(prefix);
append_intermediate_ends();
}
bt_list_producer::bt_list_producer(bt_list_producer&& other)
: data{std::move(other.data)}, buffer{other.buffer}, from{other.from}, to{other.to} {
if (other.has_child) throw std::logic_error{"Cannot move bt_list/dict_producer with active sublists/subdicts"};
var::visit([](auto& x) {
if constexpr (!std::is_same_v<buf_span&, decltype(x)>)
x = nullptr;
}, other.data);
}
bt_list_producer::bt_list_producer(char* begin, char* end)
: data{buf_span{begin, end}}, buffer{*std::get_if<buf_span>(&data)}, from{buffer.first} {
buffer_append("l"sv);
append_intermediate_ends();
}
bt_list_producer::~bt_list_producer() {
var::visit([this](auto& x) {
if constexpr (!std::is_same_v<buf_span&, decltype(x)>) {
if (!x)
return;
assert(!has_child);
assert(x->has_child);
x->has_child = false;
// We've already written the intermediate 'e', so just increment the buffer to
// finalize it.
buffer.first++;
}
}, data);
}
void bt_list_producer::append(std::string_view data) {
if (has_child) throw std::logic_error{"Cannot append to list when a sublist is active"};
append_impl(data);
append_intermediate_ends();
}
bt_list_producer bt_list_producer::append_list() {
if (has_child) throw std::logic_error{"Cannot call append_list while another nested list/dict is active"};
return bt_list_producer{this};
}
bt_dict_producer bt_list_producer::append_dict() {
if (has_child) throw std::logic_error{"Cannot call append_dict while another nested list/dict is active"};
return bt_dict_producer{this};
}
void bt_list_producer::buffer_append(std::string_view d, bool advance) {
var::visit([d, advance, this](auto& x) {
if constexpr (std::is_same_v<buf_span&, decltype(x)>) {
size_t avail = std::distance(x.first, x.second);
if (d.size() > avail)
throw std::length_error{"Cannot write bt_producer: buffer size exceeded"};
std::copy(d.begin(), d.end(), x.first);
to = x.first + d.size();
if (advance)
x.first += d.size();
} else {
x->buffer_append(d, advance);
}
}, data);
}
static constexpr std::string_view eee = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"sv;
void bt_list_producer::append_intermediate_ends(size_t count) {
return var::visit([this, count](auto& x) mutable {
if constexpr (std::is_same_v<buf_span&, decltype(x)>) {
for (; count > eee.size(); count -= eee.size())
buffer_append(eee, false);
buffer_append(eee.substr(0, count), false);
} else {
// x is a parent pointer
x->append_intermediate_ends(count + 1);
to = x->to - 1; // Our `to` should be one 'e' before our parent's `to`.
}
}, data);
}
void bt_list_producer::append_impl(std::string_view s) {
char buf[21]; // length + ':'
auto* ptr = write_integer(s.size(), buf);
*ptr++ = ':';
buffer_append({buf, static_cast<size_t>(ptr-buf)});
buffer_append(s);
}
// Subdict constructors
bt_dict_producer::bt_dict_producer(bt_list_producer* parent) : bt_list_producer{parent, "d"sv} {}
bt_dict_producer::bt_dict_producer(bt_dict_producer* parent) : bt_list_producer{parent, "d"sv} {}
#ifndef NDEBUG
void bt_dict_producer::check_incrementing_key(size_t size) {
std::string_view this_key{buffer.first - size, size};
assert(!last_key.data() || this_key > last_key);
last_key = this_key;
}
#endif
bt_dict_producer bt_dict_producer::append_dict(std::string_view key) {
if (has_child) throw std::logic_error{"Cannot call append_dict while another nested list/dict is active"};
append_impl(key);
check_incrementing_key(key.size());
return bt_dict_producer{this};
}
bt_list_producer bt_dict_producer::append_list(std::string_view key) {
if (has_child) throw std::logic_error{"Cannot call append_list while another nested list/dict is active"};
append_impl(key);
check_incrementing_key(key.size());
return bt_list_producer{this};
}
} // namespace oxenmq

905
oxenmq/bt_serialize.h
View File

@ -28,902 +28,21 @@
#pragma once
#include <vector>
#include <functional>
#include <cstring>
#include <sstream>
#include <ostream>
#include <string>
#include <string_view>
#include "variant.h"
#include <cstdint>
#include <limits>
#include <stdexcept>
#include <type_traits>
#include <utility>
#include <tuple>
#include <algorithm>
// Compatibility shim for oxenc includes
#include "bt_value.h"
#include <oxenc/bt_serialize.h>
namespace oxenmq {
using namespace std::literals;
/** \file
* OxenMQ serialization for internal commands is very simple: we support two primitive types,
* strings and integers, and two container types, lists and dicts with string keys. On the wire
* these go in BitTorrent byte encoding as described in BEP-0003
* (https://www.bittorrent.org/beps/bep_0003.html#bencoding).
*
* On the C++ side, on input we allow strings, integral types, STL-like containers of these types,
* and STL-like containers of pairs with a string first value and any of these types as second
* value. We also accept std::variants of these.
*
* One minor deviation from BEP-0003 is that we don't support serializing values that don't fit in a
* 64-bit integer (BEP-0003 specifies arbitrary precision integers).
*
* On deserialization we can either deserialize into a special bt_value type supports everything
* (with arbitrary nesting), or we can fill a container of your given type (though this fails if the
* container isn't compatible with the deserialized data).
*
* There is also a stream deserialization that allows you to deserialize without needing heap
* allocations (as long as you know the precise data structure layout).
*/
/// Exception throw if deserialization fails
class bt_deserialize_invalid : public std::invalid_argument {
using std::invalid_argument::invalid_argument;
};
/// A more specific subclass that is thown if the serialization type is an initial mismatch: for
/// example, trying deserializing an int but the next thing in input is a list. This is not,
/// however, thrown if the type initially looks fine but, say, a nested serialization fails. This
/// error will only be thrown when the input stream has not been advanced (and so can be tried for a
/// different type).
class bt_deserialize_invalid_type : public bt_deserialize_invalid {
using bt_deserialize_invalid::bt_deserialize_invalid;
};
namespace detail {
/// Reads digits into an unsigned 64-bit int.
uint64_t extract_unsigned(std::string_view& s);
// (Provide non-constant lvalue and rvalue ref functions so that we only accept explicit
// string_views but not implicitly converted ones)
inline uint64_t extract_unsigned(std::string_view&& s) { return extract_unsigned(s); }
// Fallback base case; we only get here if none of the partial specializations below work
template <typename T, typename SFINAE = void>
struct bt_serialize { static_assert(!std::is_same_v<T, T>, "Cannot serialize T: unsupported type for bt serialization"); };
template <typename T, typename SFINAE = void>
struct bt_deserialize { static_assert(!std::is_same_v<T, T>, "Cannot deserialize T: unsupported type for bt deserialization"); };
/// Checks that we aren't at the end of a string view and throws if we are.
inline void bt_need_more(const std::string_view &s) {
if (s.empty())
throw bt_deserialize_invalid{"Unexpected end of string while deserializing"};
}
/// Deserializes a signed or unsigned 64-bit integer from a string. Sets the second bool to true
/// iff the value read was negative, false if positive; in either case the unsigned value is return
/// in .first. Throws an exception if the read value doesn't fit in a int64_t (if negative) or a
/// uint64_t (if positive). Removes consumed characters from the string_view.
std::pair<uint64_t, bool> bt_deserialize_integer(std::string_view& s);
/// Integer specializations
template <typename T>
struct bt_serialize<T, std::enable_if_t<std::is_integral_v<T>>> {
static_assert(sizeof(T) <= sizeof(uint64_t), "Serialization of integers larger than uint64_t is not supported");
void operator()(std::ostream &os, const T &val) {
// Cast 1-byte types to a larger type to avoid iostream interpreting them as single characters
using output_type = std::conditional_t<(sizeof(T) > 1), T, std::conditional_t<std::is_signed_v<T>, int, unsigned>>;
os << 'i' << static_cast<output_type>(val) << 'e';
}
};
template <typename T>
struct bt_deserialize<T, std::enable_if_t<std::is_integral_v<T>>> {
void operator()(std::string_view& s, T &val) {
constexpr uint64_t umax = static_cast<uint64_t>(std::numeric_limits<T>::max());
constexpr int64_t smin = static_cast<int64_t>(std::numeric_limits<T>::min());
auto [magnitude, negative] = bt_deserialize_integer(s);
if (std::is_signed_v<T>) {
if (!negative) {
if (magnitude > umax)
throw bt_deserialize_invalid("Integer deserialization failed: found too-large value " + std::to_string(magnitude) + " > " + std::to_string(umax));
val = static_cast<T>(magnitude);
} else {
auto sval = -static_cast<int64_t>(magnitude);
if (!std::is_same_v<T, int64_t> && sval < smin)
throw bt_deserialize_invalid("Integer deserialization failed: found too-low value " + std::to_string(sval) + " < " + std::to_string(smin));
val = static_cast<T>(sval);
}
} else {
if (negative)
throw bt_deserialize_invalid("Integer deserialization failed: found negative value -" + std::to_string(magnitude) + " but type is unsigned");
if (!std::is_same_v<T, uint64_t> && magnitude > umax)
throw bt_deserialize_invalid("Integer deserialization failed: found too-large value " + std::to_string(magnitude) + " > " + std::to_string(umax));
val = static_cast<T>(magnitude);
}
}
};
extern template struct bt_deserialize<int64_t>;
extern template struct bt_deserialize<uint64_t>;
template <>
struct bt_serialize<std::string_view> {
void operator()(std::ostream &os, const std::string_view &val) { os << val.size(); os.put(':'); os.write(val.data(), val.size()); }
};
template <>
struct bt_deserialize<std::string_view> {
void operator()(std::string_view& s, std::string_view& val);
};
/// String specialization
template <>
struct bt_serialize<std::string> {
void operator()(std::ostream &os, const std::string &val) { bt_serialize<std::string_view>{}(os, val); }
};
template <>
struct bt_deserialize<std::string> {
void operator()(std::string_view& s, std::string& val) { std::string_view view; bt_deserialize<std::string_view>{}(s, view); val = {view.data(), view.size()}; }
};
/// char * and string literals -- we allow serialization for convenience, but not deserialization
template <>
struct bt_serialize<char *> {
void operator()(std::ostream &os, const char *str) { bt_serialize<std::string_view>{}(os, {str, std::strlen(str)}); }
};
template <size_t N>
struct bt_serialize<char[N]> {
void operator()(std::ostream &os, const char *str) { bt_serialize<std::string_view>{}(os, {str, N-1}); }
};
/// Partial dict validity; we don't check the second type for serializability, that will be handled
/// via the base case static_assert if invalid.
template <typename T, typename = void> struct is_bt_input_dict_container_impl : std::false_type {};
template <typename T>
struct is_bt_input_dict_container_impl<T, std::enable_if_t<
std::is_same_v<std::string, std::remove_cv_t<typename T::value_type::first_type>> ||
std::is_same_v<std::string_view, std::remove_cv_t<typename T::value_type::first_type>>,
std::void_t<typename T::const_iterator /* is const iterable */,
typename T::value_type::second_type /* has a second type */>>>
: std::true_type {};
/// Determines whether the type looks like something we can insert into (using `v.insert(v.end(), x)`)
template <typename T, typename = void> struct is_bt_insertable_impl : std::false_type {};
template <typename T>
struct is_bt_insertable_impl<T,
std::void_t<decltype(std::declval<T>().insert(std::declval<T>().end(), std::declval<typename T::value_type>()))>>
: std::true_type {};
template <typename T>
constexpr bool is_bt_insertable = is_bt_insertable_impl<T>::value;
/// Determines whether the given type looks like a compatible map (i.e. has std::string keys) that
/// we can insert into.
template <typename T, typename = void> struct is_bt_output_dict_container_impl : std::false_type {};
template <typename T>
struct is_bt_output_dict_container_impl<T, std::enable_if_t<
std::is_same_v<std::string, std::remove_cv_t<typename T::value_type::first_type>> && is_bt_insertable<T>,
std::void_t<typename T::value_type::second_type /* has a second type */>>>
: std::true_type {};
template <typename T>
constexpr bool is_bt_output_dict_container = is_bt_output_dict_container_impl<T>::value;
template <typename T>
constexpr bool is_bt_input_dict_container = is_bt_output_dict_container_impl<T>::value;
// Sanity checks:
static_assert(is_bt_input_dict_container<bt_dict>);
static_assert(is_bt_output_dict_container<bt_dict>);
/// Specialization for a dict-like container (such as an unordered_map). We accept anything for a
/// dict that is const iterable over something that looks like a pair with std::string for first
/// value type. The value (i.e. second element of the pair) also must be serializable.
template <typename T>
struct bt_serialize<T, std::enable_if_t<is_bt_input_dict_container<T>>> {
using second_type = typename T::value_type::second_type;
using ref_pair = std::reference_wrapper<const typename T::value_type>;
void operator()(std::ostream &os, const T &dict) {
os << 'd';
std::vector<ref_pair> pairs;
pairs.reserve(dict.size());
for (const auto &pair : dict)
pairs.emplace(pairs.end(), pair);
std::sort(pairs.begin(), pairs.end(), [](ref_pair a, ref_pair b) { return a.get().first < b.get().first; });
for (auto &ref : pairs) {
bt_serialize<std::string>{}(os, ref.get().first);
bt_serialize<second_type>{}(os, ref.get().second);
}
os << 'e';
}
};
template <typename T>
struct bt_deserialize<T, std::enable_if_t<is_bt_output_dict_container<T>>> {
using second_type = typename T::value_type::second_type;
void operator()(std::string_view& s, T& dict) {
// Smallest dict is 2 bytes "de", for an empty dict.
if (s.size() < 2) throw bt_deserialize_invalid("Deserialization failed: end of string found where dict expected");
if (s[0] != 'd') throw bt_deserialize_invalid_type("Deserialization failed: expected 'd', found '"s + s[0] + "'"s);
s.remove_prefix(1);
dict.clear();
bt_deserialize<std::string> key_deserializer;
bt_deserialize<second_type> val_deserializer;
while (!s.empty() && s[0] != 'e') {
std::string key;
second_type val;
key_deserializer(s, key);
val_deserializer(s, val);
dict.insert(dict.end(), typename T::value_type{std::move(key), std::move(val)});
}
if (s.empty())
throw bt_deserialize_invalid("Deserialization failed: encountered end of string before dict was finished");
s.remove_prefix(1); // Consume the 'e'
}
};
/// Accept anything that looks iterable; value serialization validity isn't checked here (it fails
/// via the base case static assert).
template <typename T, typename = void> struct is_bt_input_list_container_impl : std::false_type {};
template <typename T>
struct is_bt_input_list_container_impl<T, std::enable_if_t<
!std::is_same_v<T, std::string> && !std::is_same_v<T, std::string_view> && !is_bt_input_dict_container<T>,
std::void_t<typename T::const_iterator, typename T::value_type>>>
: std::true_type {};
template <typename T, typename = void> struct is_bt_output_list_container_impl : std::false_type {};
template <typename T>
struct is_bt_output_list_container_impl<T, std::enable_if_t<
!std::is_same_v<T, std::string> && !is_bt_output_dict_container<T> && is_bt_insertable<T>>>
: std::true_type {};
template <typename T>
constexpr bool is_bt_output_list_container = is_bt_output_list_container_impl<T>::value;
template <typename T>
constexpr bool is_bt_input_list_container = is_bt_input_list_container_impl<T>::value;
// Sanity checks:
static_assert(is_bt_input_list_container<bt_list>);
static_assert(is_bt_output_list_container<bt_list>);
/// List specialization
template <typename T>
struct bt_serialize<T, std::enable_if_t<is_bt_input_list_container<T>>> {
void operator()(std::ostream& os, const T& list) {
os << 'l';
for (const auto &v : list)
bt_serialize<std::remove_cv_t<typename T::value_type>>{}(os, v);
os << 'e';
}
};
template <typename T>
struct bt_deserialize<T, std::enable_if_t<is_bt_output_list_container<T>>> {
using value_type = typename T::value_type;
void operator()(std::string_view& s, T& list) {
// Smallest list is 2 bytes "le", for an empty list.
if (s.size() < 2) throw bt_deserialize_invalid("Deserialization failed: end of string found where list expected");
if (s[0] != 'l') throw bt_deserialize_invalid_type("Deserialization failed: expected 'l', found '"s + s[0] + "'"s);
s.remove_prefix(1);
list.clear();
bt_deserialize<value_type> deserializer;
while (!s.empty() && s[0] != 'e') {
value_type v;
deserializer(s, v);
list.insert(list.end(), std::move(v));
}
if (s.empty())
throw bt_deserialize_invalid("Deserialization failed: encountered end of string before list was finished");
s.remove_prefix(1); // Consume the 'e'
}
};
/// Serializes a tuple or pair of serializable values (as a list on the wire)
/// Common implementation for both tuple and pair:
template <template<typename...> typename Tuple, typename... T>
struct bt_serialize_tuple {
private:
template <size_t... Is>
void operator()(std::ostream& os, const Tuple<T...>& elems, std::index_sequence<Is...>) {
os << 'l';
(bt_serialize<T>{}(os, std::get<Is>(elems)), ...);
os << 'e';
}
public:
void operator()(std::ostream& os, const Tuple<T...>& elems) {
operator()(os, elems, std::index_sequence_for<T...>{});
}
};
template <template<typename...> typename Tuple, typename... T>
struct bt_deserialize_tuple {
private:
template <size_t... Is>
void operator()(std::string_view& s, Tuple<T...>& elems, std::index_sequence<Is...>) {
// Smallest list is 2 bytes "le", for an empty list.
if (s.size() < 2) throw bt_deserialize_invalid("Deserialization failed: end of string found where tuple expected");
if (s[0] != 'l') throw bt_deserialize_invalid_type("Deserialization of tuple failed: expected 'l', found '"s + s[0] + "'"s);
s.remove_prefix(1);
(bt_deserialize<T>{}(s, std::get<Is>(elems)), ...);
if (s.empty())
throw bt_deserialize_invalid("Deserialization failed: encountered end of string before tuple was finished");
if (s[0] != 'e')
throw bt_deserialize_invalid("Deserialization failed: expected end of tuple but found something else");
s.remove_prefix(1); // Consume the 'e'
}
public:
void operator()(std::string_view& s, Tuple<T...>& elems) {
operator()(s, elems, std::index_sequence_for<T...>{});
}
};
template <typename... T>
struct bt_serialize<std::tuple<T...>> : bt_serialize_tuple<std::tuple, T...> {};
template <typename... T>
struct bt_deserialize<std::tuple<T...>> : bt_deserialize_tuple<std::tuple, T...> {};
template <typename S, typename T>
struct bt_serialize<std::pair<S, T>> : bt_serialize_tuple<std::pair, S, T> {};
template <typename S, typename T>
struct bt_deserialize<std::pair<S, T>> : bt_deserialize_tuple<std::pair, S, T> {};
template <typename T>
inline constexpr bool is_bt_tuple = false;
template <typename... T>
inline constexpr bool is_bt_tuple<std::tuple<T...>> = true;
template <typename S, typename T>
inline constexpr bool is_bt_tuple<std::pair<S, T>> = true;
template <typename T>
constexpr bool is_bt_deserializable = std::is_same_v<T, std::string> || std::is_integral_v<T> ||
is_bt_output_dict_container<T> || is_bt_output_list_container<T> || is_bt_tuple<T>;
// General template and base case; this base will only actually be invoked when Ts... is empty,
// which means we reached the end without finding any variant type capable of holding the value.
template <typename SFINAE, typename Variant, typename... Ts>
struct bt_deserialize_try_variant_impl {
void operator()(std::string_view&, Variant&) {
throw bt_deserialize_invalid("Deserialization failed: could not deserialize value into any variant type");
}
};
template <typename... Ts, typename Variant>
void bt_deserialize_try_variant(std::string_view& s, Variant& variant) {
bt_deserialize_try_variant_impl<void, Variant, Ts...>{}(s, variant);
}
template <typename Variant, typename T, typename... Ts>
struct bt_deserialize_try_variant_impl<std::enable_if_t<is_bt_deserializable<T>>, Variant, T, Ts...> {
void operator()(std::string_view& s, Variant& variant) {
if ( is_bt_output_list_container<T> ? s[0] == 'l' :
is_bt_tuple<T> ? s[0] == 'l' :
is_bt_output_dict_container<T> ? s[0] == 'd' :
std::is_integral_v<T> ? s[0] == 'i' :
std::is_same_v<T, std::string> ? s[0] >= '0' && s[0] <= '9' :
false) {
T val;
bt_deserialize<T>{}(s, val);
variant = std::move(val);
} else {
bt_deserialize_try_variant<Ts...>(s, variant);
}
}
};
template <typename Variant, typename T, typename... Ts>
struct bt_deserialize_try_variant_impl<std::enable_if_t<!is_bt_deserializable<T>>, Variant, T, Ts...> {
void operator()(std::string_view& s, Variant& variant) {
// Unsupported deserialization type, skip it
bt_deserialize_try_variant<Ts...>(s, variant);
}
};
// Serialization of a variant; all variant types must be bt-serializable.
template <typename... Ts>
struct bt_serialize<std::variant<Ts...>, std::void_t<bt_serialize<Ts>...>> {
void operator()(std::ostream& os, const std::variant<Ts...>& val) {
var::visit(
[&os] (const auto& val) {
using T = std::remove_cv_t<std::remove_reference_t<decltype(val)>>;
bt_serialize<T>{}(os, val);
},
val);
}
};
// Deserialization to a variant; at least one variant type must be bt-deserializble.
template <typename... Ts>
struct bt_deserialize<std::variant<Ts...>, std::enable_if_t<(is_bt_deserializable<Ts> || ...)>> {
void operator()(std::string_view& s, std::variant<Ts...>& val) {
bt_deserialize_try_variant<Ts...>(s, val);
}
};
template <>
struct bt_serialize<bt_value> : bt_serialize<bt_variant> {};
template <>
struct bt_deserialize<bt_value> {
void operator()(std::string_view& s, bt_value& val);
};
template <typename T>
struct bt_stream_serializer {
const T &val;
explicit bt_stream_serializer(const T &val) : val{val} {}
operator std::string() const {
std::ostringstream oss;
oss << *this;
return oss.str();
}
};
template <typename T>
std::ostream &operator<<(std::ostream &os, const bt_stream_serializer<T> &s) {
bt_serialize<T>{}(os, s.val);
return os;
}
} // namespace detail
/// Returns a wrapper around a value reference that can serialize the value directly to an output
/// stream. This class is intended to be used inline (i.e. without being stored) as in:
///
/// std::list<int> my_list{{1,2,3}};
/// std::cout << bt_serializer(my_list);
///
/// While it is possible to store the returned object and use it, such as:
///
/// auto encoded = bt_serializer(42);
/// std::cout << encoded;
///
/// this approach is not generally recommended: the returned object stores a reference to the
/// passed-in type, which may not survive. If doing this note that it is the caller's
/// responsibility to ensure the serializer is not used past the end of the lifetime of the value
/// being serialized.
///
/// Also note that serializing directly to an output stream is more efficient as no intermediate
/// string containing the entire serialization has to be constructed.
///
template <typename T>
detail::bt_stream_serializer<T> bt_serializer(const T &val) { return detail::bt_stream_serializer<T>{val}; }
/// Serializes the given value into a std::string.
///
/// int number = 42;
/// std::string encoded = bt_serialize(number);
/// // Equivalent:
/// //auto encoded = (std::string) bt_serialize(number);
///
/// This takes any serializable type: integral types, strings, lists of serializable types, and
/// string->value maps of serializable types.
template <typename T>
std::string bt_serialize(const T &val) { return bt_serializer(val); }
/// Deserializes the given string view directly into `val`. Usage:
///
/// std::string encoded = "i42e";
/// int value;
/// bt_deserialize(encoded, value); // Sets value to 42
///
template <typename T, std::enable_if_t<!std::is_const_v<T>, int> = 0>
void bt_deserialize(std::string_view s, T& val) {
return detail::bt_deserialize<T>{}(s, val);
}
/// Deserializes the given string_view into a `T`, which is returned.
///
/// std::string encoded = "li1ei2ei3ee"; // bt-encoded list of ints: [1,2,3]
/// auto mylist = bt_deserialize<std::list<int>>(encoded);
///
template <typename T>
T bt_deserialize(std::string_view s) {
T val;
bt_deserialize(s, val);
return val;
}
/// Deserializes the given value into a generic `bt_value` type (wrapped std::variant) which is
/// capable of holding all possible BT-encoded values (including recursion).
///
/// Example:
///
/// std::string encoded = "i42e";
/// auto val = bt_get(encoded);
/// int v = get_int<int>(val); // fails unless the encoded value was actually an integer that
/// // fits into an `int`
///
inline bt_value bt_get(std::string_view s) {
return bt_deserialize<bt_value>(s);
}
/// Helper functions to extract a value of some integral type from a bt_value which contains either
/// a int64_t or uint64_t. Does range checking, throwing std::overflow_error if the stored value is
/// outside the range of the target type.
///
/// Example:
///
/// std::string encoded = "i123456789e";
/// auto val = bt_get(encoded);
/// auto v = get_int<uint32_t>(val); // throws if the decoded value doesn't fit in a uint32_t
template <typename IntType, std::enable_if_t<std::is_integral_v<IntType>, int> = 0>
IntType get_int(const bt_value &v) {
if (auto* value = std::get_if<uint64_t>(&v)) {
if constexpr (!std::is_same_v<IntType, uint64_t>)
if (*value > static_cast<uint64_t>(std::numeric_limits<IntType>::max()))
throw std::overflow_error("Unable to extract integer value: stored value is too large for the requested type");
return static_cast<IntType>(*value);
}
int64_t value = var::get<int64_t>(v); // throws if no int contained
if constexpr (!std::is_same_v<IntType, int64_t>)
if (value > static_cast<int64_t>(std::numeric_limits<IntType>::max())
|| value < static_cast<int64_t>(std::numeric_limits<IntType>::min()))
throw std::overflow_error("Unable to extract integer value: stored value is outside the range of the requested type");
return static_cast<IntType>(value);
}
namespace detail {
template <typename Tuple, size_t... Is>
void get_tuple_impl(Tuple& t, const bt_list& l, std::index_sequence<Is...>);
}
/// Converts a bt_list into the given template std::tuple or std::pair. Throws a
/// std::invalid_argument if the list has the wrong size or wrong element types. Supports recursion
/// (i.e. if the tuple itself contains tuples or pairs). The tuple (or nested tuples) may only
/// contain integral types, strings, string_views, bt_list, bt_dict, and tuples/pairs of those.
template <typename Tuple>
Tuple get_tuple(const bt_list& x) {
Tuple t;
detail::get_tuple_impl(t, x, std::make_index_sequence<std::tuple_size_v<Tuple>>{});
return t;
}
template <typename Tuple>
Tuple get_tuple(const bt_value& x) {
return get_tuple<Tuple>(var::get<bt_list>(static_cast<const bt_variant&>(x)));
}
namespace detail {
template <typename T, typename It>
void get_tuple_impl_one(T& t, It& it) {
const bt_variant& v = *it++;
if constexpr (std::is_integral_v<T>) {
t = oxenmq::get_int<T>(v);
} else if constexpr (is_bt_tuple<T>) {
if (std::holds_alternative<bt_list>(v))
throw std::invalid_argument{"Unable to convert tuple: cannot create sub-tuple from non-bt_list"};
t = get_tuple<T>(var::get<bt_list>(v));
} else if constexpr (std::is_same_v<std::string, T> || std::is_same_v<std::string_view, T>) {
// If we request a string/string_view, we might have the other one and need to copy/view it.
if (std::holds_alternative<std::string_view>(v))
t = var::get<std::string_view>(v);
else
t = var::get<std::string>(v);
} else {
t = var::get<T>(v);
}
}
template <typename Tuple, size_t... Is>
void get_tuple_impl(Tuple& t, const bt_list& l, std::index_sequence<Is...>) {
if (l.size() != sizeof...(Is))
throw std::invalid_argument{"Unable to convert tuple: bt_list has wrong size"};
auto it = l.begin();
(get_tuple_impl_one(std::get<Is>(t), it), ...);
}
} // namespace detail
class bt_dict_consumer;
/// Class that allows you to walk through a bt-encoded list in memory without copying or allocating
/// memory. It accesses existing memory directly and so the caller must ensure that the referenced
/// memory stays valid for the lifetime of the bt_list_consumer object.
class bt_list_consumer {
protected:
std::string_view data;
bt_list_consumer() = default;
public:
bt_list_consumer(std::string_view data_);
/// Copy constructor. Making a copy copies the current position so can be used for multipass
/// iteration through a list.
bt_list_consumer(const bt_list_consumer&) = default;
bt_list_consumer& operator=(const bt_list_consumer&) = default;
/// Get a copy of the current buffer
std::string_view current_buffer() const { return data; }
/// Returns true if the next value indicates the end of the list
bool is_finished() const { return data.front() == 'e'; }
/// Returns true if the next element looks like an encoded string
bool is_string() const { return data.front() >= '0' && data.front() <= '9'; }
/// Returns true if the next element looks like an encoded integer
bool is_integer() const { return data.front() == 'i'; }
/// Returns true if the next element looks like an encoded negative integer
bool is_negative_integer() const { return is_integer() && data.size() >= 2 && data[1] == '-'; }
/// Returns true if the next element looks like an encoded non-negative integer
bool is_unsigned_integer() const { return is_integer() && data.size() >= 2 && data[1] >= '0' && data[1] <= '9'; }
/// Returns true if the next element looks like an encoded list
bool is_list() const { return data.front() == 'l'; }
/// Returns true if the next element looks like an encoded dict
bool is_dict() const { return data.front() == 'd'; }
/// Attempt to parse the next value as a string (and advance just past it). Throws if the next
/// value is not a string.
std::string consume_string();
std::string_view consume_string_view();
/// Attempts to parse the next value as an integer (and advance just past it). Throws if the
/// next value is not an integer.
template <typename IntType>
IntType consume_integer() {
if (!is_integer()) throw bt_deserialize_invalid_type{"next value is not an integer"};
std::string_view next{data};
IntType ret;
detail::bt_deserialize<IntType>{}(next, ret);
data = next;
return ret;
}
/// Consumes a list, return it as a list-like type. Can also be used for tuples/pairs. This
/// typically requires dynamic allocation, but only has to parse the data once. Compare with
/// consume_list_data() which allows alloc-free traversal, but requires parsing twice (if the
/// contents are to be used).
template <typename T = bt_list>
T consume_list() {
T list;
consume_list(list);
return list;
}
/// Same as above, but takes a pre-existing list-like data type.
template <typename T>
void consume_list(T& list) {
if (!is_list()) throw bt_deserialize_invalid_type{"next bt value is not a list"};
std::string_view n{data};
detail::bt_deserialize<T>{}(n, list);
data = n;
}
/// Consumes a dict, return it as a dict-like type. This typically requires dynamic allocation,
/// but only has to parse the data once. Compare with consume_dict_data() which allows
/// alloc-free traversal, but requires parsing twice (if the contents are to be used).
template <typename T = bt_dict>
T consume_dict() {
T dict;
consume_dict(dict);
return dict;
}
/// Same as above, but takes a pre-existing dict-like data type.
template <typename T>
void consume_dict(T& dict) {
if (!is_dict()) throw bt_deserialize_invalid_type{"next bt value is not a dict"};
std::string_view n{data};
detail::bt_deserialize<T>{}(n, dict);
data = n;
}
/// Consumes a value without returning it.
void skip_value();
/// Attempts to parse the next value as a list and returns the string_view that contains the
/// entire thing. This is recursive into both lists and dicts and likely to be quite
/// inefficient for large, nested structures (unless the values only need to be skipped but
/// aren't separately needed). This, however, does not require dynamic memory allocation.
std::string_view consume_list_data();
/// Attempts to parse the next value as a dict and returns the string_view that contains the
/// entire thing. This is recursive into both lists and dicts and likely to be quite
/// inefficient for large, nested structures (unless the values only need to be skipped but
/// aren't separately needed). This, however, does not require dynamic memory allocation.
std::string_view consume_dict_data();
/// Shortcut for wrapping `consume_list_data()` in a new list consumer
bt_list_consumer consume_list_consumer() { return consume_list_data(); }
/// Shortcut for wrapping `consume_dict_data()` in a new dict consumer
bt_dict_consumer consume_dict_consumer();
};
/// Class that allows you to walk through key-value pairs of a bt-encoded dict in memory without
/// copying or allocating memory. It accesses existing memory directly and so the caller must
/// ensure that the referenced memory stays valid for the lifetime of the bt_dict_consumer object.
class bt_dict_consumer : private bt_list_consumer {
std::string_view key_;
/// Consume the key if not already consumed and there is a key present (rather than 'e').
/// Throws exception if what should be a key isn't a string, or if the key consumes the entire
/// data (i.e. requires that it be followed by something). Returns true if the key was consumed
/// (either now or previously and cached).
bool consume_key();
/// Clears the cached key and returns it. Must have already called consume_key directly or
/// indirectly via one of the `is_{...}` methods.
std::string_view flush_key() {
std::string_view k;
k.swap(key_);
return k;
}
public:
bt_dict_consumer(std::string_view data_);
/// Copy constructor. Making a copy copies the current position so can be used for multipass
/// iteration through a list.
bt_dict_consumer(const bt_dict_consumer&) = default;
bt_dict_consumer& operator=(const bt_dict_consumer&) = default;
/// Returns true if the next value indicates the end of the dict
bool is_finished() { return !consume_key() && data.front() == 'e'; }
/// Operator bool is an alias for `!is_finished()`
operator bool() { return !is_finished(); }
/// Returns true if the next value looks like an encoded string
bool is_string() { return consume_key() && data.front() >= '0' && data.front() <= '9'; }
/// Returns true if the next element looks like an encoded integer
bool is_integer() { return consume_key() && data.front() == 'i'; }
/// Returns true if the next element looks like an encoded negative integer
bool is_negative_integer() { return is_integer() && data.size() >= 2 && data[1] == '-'; }
/// Returns true if the next element looks like an encoded non-negative integer
bool is_unsigned_integer() { return is_integer() && data.size() >= 2 && data[1] >= '0' && data[1] <= '9'; }
/// Returns true if the next element looks like an encoded list
bool is_list() { return consume_key() && data.front() == 'l'; }
/// Returns true if the next element looks like an encoded dict
bool is_dict() { return consume_key() && data.front() == 'd'; }
/// Returns the key of the next pair. This does not have to be called; it is also returned by
/// all of the other consume_* methods. The value is cached whether called here or by some
/// other method; accessing it multiple times simple accesses the cache until the next value is
/// consumed.
std::string_view key() {
if (!consume_key())
throw bt_deserialize_invalid{"Cannot access next key: at the end of the dict"};
return key_;
}
/// Attempt to parse the next value as a string->string pair (and advance just past it). Throws
/// if the next value is not a string.
std::pair<std::string_view, std::string_view> next_string();
/// Attempts to parse the next value as an string->integer pair (and advance just past it).
/// Throws if the next value is not an integer.
template <typename IntType>
std::pair<std::string_view, IntType> next_integer() {
if (!is_integer()) throw bt_deserialize_invalid_type{"next bt dict value is not an integer"};
std::pair<std::string_view, IntType> ret;
ret.second = bt_list_consumer::consume_integer<IntType>();
ret.first = flush_key();
return ret;
}
/// Consumes a string->list pair, return it as a list-like type. This typically requires
/// dynamic allocation, but only has to parse the data once. Compare with consume_list_data()
/// which allows alloc-free traversal, but requires parsing twice (if the contents are to be
/// used).
template <typename T = bt_list>
std::pair<std::string_view, T> next_list() {
std::pair<std::string_view, T> pair;
pair.first = next_list(pair.second);
return pair;
}
/// Same as above, but takes a pre-existing list-like data type. Returns the key.
template <typename T>
std::string_view next_list(T& list) {
if (!is_list()) throw bt_deserialize_invalid_type{"next bt value is not a list"};
bt_list_consumer::consume_list(list);
return flush_key();
}
/// Consumes a string->dict pair, return it as a dict-like type. This typically requires
/// dynamic allocation, but only has to parse the data once. Compare with consume_dict_data()
/// which allows alloc-free traversal, but requires parsing twice (if the contents are to be
/// used).
template <typename T = bt_dict>
std::pair<std::string_view, T> next_dict() {
std::pair<std::string_view, T> pair;
pair.first = consume_dict(pair.second);
return pair;
}
/// Same as above, but takes a pre-existing dict-like data type. Returns the key.
template <typename T>
std::string_view next_dict(T& dict) {
if (!is_dict()) throw bt_deserialize_invalid_type{"next bt value is not a dict"};
bt_list_consumer::consume_dict(dict);
return flush_key();
}
/// Attempts to parse the next value as a string->list pair and returns the string_view that
/// contains the entire thing. This is recursive into both lists and dicts and likely to be
/// quite inefficient for large, nested structures (unless the values only need to be skipped
/// but aren't separately needed). This, however, does not require dynamic memory allocation.
std::pair<std::string_view, std::string_view> next_list_data() {
if (data.size() < 2 || !is_list()) throw bt_deserialize_invalid_type{"next bt dict value is not a list"};
return {flush_key(), bt_list_consumer::consume_list_data()};
}
/// Same as next_list_data(), but wraps the value in a bt_list_consumer for convenience
std::pair<std::string_view, bt_list_consumer> next_list_consumer() { return next_list_data(); }
/// Attempts to parse the next value as a string->dict pair and returns the string_view that
/// contains the entire thing. This is recursive into both lists and dicts and likely to be
/// quite inefficient for large, nested structures (unless the values only need to be skipped
/// but aren't separately needed). This, however, does not require dynamic memory allocation.
std::pair<std::string_view, std::string_view> next_dict_data() {
if (data.size() < 2 || !is_dict()) throw bt_deserialize_invalid_type{"next bt dict value is not a dict"};
return {flush_key(), bt_list_consumer::consume_dict_data()};
}
/// Same as next_dict_data(), but wraps the value in a bt_dict_consumer for convenience
std::pair<std::string_view, bt_dict_consumer> next_dict_consumer() { return next_dict_data(); }
/// Skips ahead until we find the first key >= the given key or reach the end of the dict.
/// Returns true if we found an exact match, false if we reached some greater value or the end.
/// If we didn't hit the end, the next `consumer_*()` call will return the key-value pair we
/// found (either the exact match or the first key greater than the requested key).
///
/// Two important notes:
///
/// - properly encoded bt dicts must have lexicographically sorted keys, and this method assumes
/// that the input is correctly sorted (and thus if we find a greater value then your key does
/// not exist).
/// - this is irreversible; you cannot returned to skipped values without reparsing. (You *can*
/// however, make a copy of the bt_dict_consumer before calling and use the copy to return to
/// the pre-skipped position).
bool skip_until(std::string_view find) {
while (consume_key() && key_ < find) {
flush_key();
skip_value();
}
return key_ == find;
}
/// The `consume_*` functions are wrappers around next_whatever that discard the returned key.
///
/// Intended for use with skip_until such as:
///
/// std::string value;
/// if (d.skip_until("key"))
/// value = d.consume_string();
///
auto consume_string_view() { return next_string().second; }
auto consume_string() { return std::string{consume_string_view()}; }
template <typename IntType>
auto consume_integer() { return next_integer<IntType>().second; }
template <typename T = bt_list>
auto consume_list() { return next_list<T>().second; }
template <typename T>
void consume_list(T& list) { next_list(list); }
template <typename T = bt_dict>
auto consume_dict() { return next_dict<T>().second; }
template <typename T>
void consume_dict(T& dict) { next_dict(dict); }
std::string_view consume_list_data() { return next_list_data().second; }
std::string_view consume_dict_data() { return next_dict_data().second; }
/// Shortcut for wrapping `consume_list_data()` in a new list consumer
bt_list_consumer consume_list_consumer() { return consume_list_data(); }
/// Shortcut for wrapping `consume_dict_data()` in a new dict consumer
bt_dict_consumer consume_dict_consumer() { return consume_dict_data(); }
};
inline bt_dict_consumer bt_list_consumer::consume_dict_consumer() { return consume_dict_data(); }
using oxenc::bt_deserialize_invalid;
using oxenc::bt_deserialize_invalid_type;
using oxenc::bt_serializer;
using oxenc::bt_serialize;
using oxenc::bt_deserialize;
using oxenc::bt_get;
using oxenc::get_int;
using oxenc::get_tuple;
using oxenc::bt_dict_consumer;
using oxenc::bt_list_consumer;
} // namespace oxenmq

83
oxenmq/bt_value.h
View File

@ -28,85 +28,18 @@
#pragma once
// This header is here to provide just the basic bt_value/bt_dict/bt_list definitions without
// needing to include the full bt_serialize.h header.
// Compatibility shim for oxenc includes
#include <map>
#include <list>
#include <cstdint>
#include <variant>
#include <string>
#include <string_view>
#include <oxenc/bt_value.h>
namespace oxenmq {
struct bt_value;
using oxenc::bt_value;
using oxenc::bt_dict;
using oxenc::bt_list;
using oxenc::bt_variant;
/// The type used to store dictionaries inside bt_value.
using bt_dict = std::map<std::string, bt_value>; // NB: unordered_map doesn't work because it can't be used with a predeclared type
/// The type used to store list items inside bt_value.
using bt_list = std::list<bt_value>;
/// The basic variant that can hold anything (recursively).
using bt_variant = std::variant<
std::string,
std::string_view,
int64_t,
uint64_t,
bt_list,
bt_dict
>;
#ifdef __cpp_lib_remove_cvref // C++20
using std::remove_cvref_t;
#else
template <typename T>
using remove_cvref_t = std::remove_cv_t<std::remove_reference_t<T>>;
#endif
template <typename T, typename Variant>
struct has_alternative;
template <typename T, typename... V>
struct has_alternative<T, std::variant<V...>> : std::bool_constant<(std::is_same_v<T, V> || ...)> {};
template <typename T, typename Variant>
constexpr bool has_alternative_v = has_alternative<T, Variant>::value;
namespace detail {
template <typename Tuple, size_t... Is>
bt_list tuple_to_list(const Tuple& tuple, std::index_sequence<Is...>) {
return {{bt_value{std::get<Is>(tuple)}...}};
}
template <typename T> constexpr bool is_tuple = false;
template <typename... T> constexpr bool is_tuple<std::tuple<T...>> = true;
template <typename S, typename T> constexpr bool is_tuple<std::pair<S, T>> = true;
}
/// Recursive generic type that can fully represent everything valid for a BT serialization.
/// This is basically just an empty wrapper around the std::variant, except we add some extra
/// converting constructors:
/// - integer constructors so that any unsigned value goes to the uint64_t and any signed value goes
/// to the int64_t.
/// - std::tuple and std::pair constructors that build a bt_list out of the tuple/pair elements.
struct bt_value : bt_variant {
using bt_variant::bt_variant;
using bt_variant::operator=;
template <typename T, typename U = std::remove_reference_t<T>, std::enable_if_t<std::is_integral_v<U> && std::is_unsigned_v<U>, int> = 0>
bt_value(T&& uint) : bt_variant{static_cast<uint64_t>(uint)} {}
template <typename T, typename U = std::remove_reference_t<T>, std::enable_if_t<std::is_integral_v<U> && std::is_signed_v<U>, int> = 0>
bt_value(T&& sint) : bt_variant{static_cast<int64_t>(sint)} {}
template <typename... T>
bt_value(const std::tuple<T...>& tuple) : bt_variant{detail::tuple_to_list(tuple, std::index_sequence_for<T...>{})} {}
template <typename S, typename T>
bt_value(const std::pair<S, T>& pair) : bt_variant{detail::tuple_to_list(pair, std::index_sequence_for<S, T>{})} {}
template <typename T, typename U = std::remove_reference_t<T>, std::enable_if_t<!std::is_integral_v<U> && !detail::is_tuple<U>, int> = 0>
bt_value(T&& v) : bt_variant{std::forward<T>(v)} {}
bt_value(const char* s) : bt_value{std::string_view{s}} {}
};
using oxenc::has_alternative;
using oxenc::has_alternative_v;
}

28
oxenmq/byte_type.h
View File

@ -1,28 +0,0 @@
#pragma once
// Specializations for assigning from a char into an output iterator, used by hex/base32z/base64
// decoding to bytes.
#include <iterator>
#include <type_traits>
namespace oxenmq::detail {
// Fallback - we just try a char
template <typename OutputIt, typename = void>
struct byte_type { using type = char; };
// Support for things like std::back_inserter:
template <typename OutputIt>
struct byte_type<OutputIt, std::void_t<typename OutputIt::container_type>> {
using type = typename OutputIt::container_type::value_type; };
// iterator, raw pointers:
template <typename OutputIt>
struct byte_type<OutputIt, std::enable_if_t<std::is_reference_v<typename std::iterator_traits<OutputIt>::reference>>> {
using type = std::remove_reference_t<typename std::iterator_traits<OutputIt>::reference>; };
template <typename OutputIt>
using byte_type_t = typename byte_type<OutputIt>::type;
}

27
oxenmq/connections.cpp
View File

@ -1,12 +1,12 @@
#include "oxenmq.h"
#include "oxenmq-internal.h"
#include "hex.h"
#include <oxenc/hex.h>
namespace oxenmq {
std::ostream& operator<<(std::ostream& o, const ConnectionID& conn) {
if (!conn.pk.empty())
return o << (conn.sn() ? "SN " : "non-SN authenticated remote ") << to_hex(conn.pk);
return o << (conn.sn() ? "SN " : "non-SN authenticated remote ") << oxenc::to_hex(conn.pk);
else
return o << "unauthenticated remote [" << conn.id << "]";
}
@ -75,7 +75,7 @@ void OxenMQ::setup_incoming_socket(zmq::socket_t& listener, bool curve, std::str
setup_external_socket(listener);
listener.set(zmq::sockopt::zap_domain, bt_serialize(bind_index));
listener.set(zmq::sockopt::zap_domain, oxenc::bt_serialize(bind_index));
if (curve) {
listener.set(zmq::sockopt::curve_server, true);
listener.set(zmq::sockopt::curve_publickey, pubkey);
@ -100,7 +100,7 @@ ConnectionID OxenMQ::connect_remote(std::string_view remote, ConnectSuccess on_c
}
void OxenMQ::disconnect(ConnectionID id, std::chrono::milliseconds linger) {
detail::send_control(get_control_socket(), "DISCONNECT", bt_serialize<bt_dict>({
detail::send_control(get_control_socket(), "DISCONNECT", oxenc::bt_serialize<oxenc::bt_dict>({
{"conn_id", id.id},
{"linger_ms", linger.count()},
{"pubkey", id.pk},
@ -122,7 +122,7 @@ OxenMQ::proxy_connect_sn(std::string_view remote, std::string_view connect_hint,
}
if (peer) {
OMQ_TRACE("proxy asked to connect to ", to_hex(remote), "; reusing existing connection");
OMQ_TRACE("proxy asked to connect to ", oxenc::to_hex(remote), "; reusing existing connection");
if (peer->route.empty() /* == outgoing*/) {
if (peer->idle_expiry < keep_alive) {
OMQ_LOG(debug, "updating existing outgoing peer connection idle expiry time from ",
@ -138,7 +138,7 @@ OxenMQ::proxy_connect_sn(std::string_view remote, std::string_view connect_hint,
}
// No connection so establish a new one
OMQ_LOG(debug, "proxy establishing new outbound connection to ", to_hex(remote));
OMQ_LOG(debug, "proxy establishing new outbound connection to ", oxenc::to_hex(remote));
std::string addr;
bool to_self = false && remote == pubkey; // FIXME; need to use a separate listening socket for this, otherwise we can't easily
// tell it wasn't from a remote.
@ -153,12 +153,12 @@ OxenMQ::proxy_connect_sn(std::string_view remote, std::string_view connect_hint,
OMQ_LOG(debug, "using connection hint ", connect_hint);
if (addr.empty()) {
OMQ_LOG(error, "peer lookup failed for ", to_hex(remote));
OMQ_LOG(error, "peer lookup failed for ", oxenc::to_hex(remote));
return {nullptr, ""s};
}
}
OMQ_LOG(debug, to_hex(pubkey), " (me) connecting to ", addr, " to reach ", to_hex(remote));
OMQ_LOG(debug, oxenc::to_hex(pubkey), " (me) connecting to ", addr, " to reach ", oxenc::to_hex(remote));
zmq::socket_t socket{context, zmq::socket_type::dealer};
setup_outgoing_socket(socket, remote, use_ephemeral_routing_id);
try {
@ -183,7 +183,7 @@ OxenMQ::proxy_connect_sn(std::string_view remote, std::string_view connect_hint,
return {&it->second, ""s};
}
std::pair<zmq::socket_t *, std::string> OxenMQ::proxy_connect_sn(bt_dict_consumer data) {
std::pair<zmq::socket_t *, std::string> OxenMQ::proxy_connect_sn(oxenc::bt_dict_consumer data) {
std::string_view hint, remote_pk;
std::chrono::milliseconds keep_alive;
bool optional = false, incoming_only = false, outgoing_only = false, ephemeral_rid = EPHEMERAL_ROUTING_ID;
@ -278,7 +278,7 @@ void OxenMQ::proxy_conn_cleanup() {
for (auto it = pending_requests.begin(); it != pending_requests.end(); ) {
auto& callback = it->second;
if (callback.first < now) {
OMQ_LOG(debug, "pending request ", to_hex(it->first), " expired, invoking callback with failure status and removing");
OMQ_LOG(debug, "pending request ", oxenc::to_hex(it->first), " expired, invoking callback with failure status and removing");
job([callback = std::move(callback.second)] { callback(false, {{"TIMEOUT"s}}); });
it = pending_requests.erase(it);
} else {
@ -289,7 +289,7 @@ void OxenMQ::proxy_conn_cleanup() {
OMQ_TRACE("done proxy connections cleanup");
};
void OxenMQ::proxy_connect_remote(bt_dict_consumer data) {
void OxenMQ::proxy_connect_remote(oxenc::bt_dict_consumer data) {
AuthLevel auth_level = AuthLevel::none;
long long conn_id = -1;
ConnectSuccess on_connect;
@ -321,7 +321,8 @@ void OxenMQ::proxy_connect_remote(bt_dict_consumer data) {
if (conn_id == -1 || remote.empty())
throw std::runtime_error("Internal error: CONNECT_REMOTE proxy command missing required 'conn_id' and/or 'remote' value");
OMQ_LOG(debug, "Establishing remote connection to ", remote, remote_pubkey.empty() ? " (NULL auth)" : " via CURVE expecting pubkey " + to_hex(remote_pubkey));
OMQ_LOG(debug, "Establishing remote connection to ", remote,
remote_pubkey.empty() ? " (NULL auth)" : " via CURVE expecting pubkey " + oxenc::to_hex(remote_pubkey));
zmq::socket_t sock{context, zmq::socket_type::dealer};
try {
@ -349,7 +350,7 @@ void OxenMQ::proxy_connect_remote(bt_dict_consumer data) {
peer.activity();
}
void OxenMQ::proxy_disconnect(bt_dict_consumer data) {
void OxenMQ::proxy_disconnect(oxenc::bt_dict_consumer data) {
ConnectionID connid{-1};
std::chrono::milliseconds linger = 1s;

6
oxenmq/connections.h
View File

@ -1,6 +1,6 @@
#pragma once
#include "auth.h"
#include "bt_value.h"
#include <oxenc/bt_value.h>
#include <string_view>
#include <iosfwd>
#include <stdexcept>
@ -14,7 +14,7 @@ struct ConnectionID;
namespace detail {
template <typename... T>
bt_dict build_send(ConnectionID to, std::string_view cmd, T&&... opts);
oxenc::bt_dict build_send(ConnectionID to, std::string_view cmd, T&&... opts);
}
/// Opaque data structure representing a connection which supports ==, !=, < and std::hash. For
@ -80,7 +80,7 @@ private:
friend class OxenMQ;
friend struct std::hash<ConnectionID>;
template <typename... T>
friend bt_dict detail::build_send(ConnectionID to, std::string_view cmd, T&&... opts);
friend oxenc::bt_dict detail::build_send(ConnectionID to, std::string_view cmd, T&&... opts);
friend std::ostream& operator<<(std::ostream& o, const ConnectionID& conn);
};

231
oxenmq/hex.h
View File

@ -27,226 +27,21 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <string>
#include <string_view>
#include <array>
#include <iterator>
#include <cassert>
#include "byte_type.h"
#include <oxenc/hex.h>
// Compatibility shim for oxenc includes
namespace oxenmq {
namespace detail {
/// Compile-time generated lookup tables hex conversion
struct hex_table {
char from_hex_lut[256];
char to_hex_lut[16];
constexpr hex_table() noexcept : from_hex_lut{}, to_hex_lut{} {
for (unsigned char c = 0; c < 10; c++) {
from_hex_lut[(unsigned char)('0' + c)] = 0 + c;
to_hex_lut[ (unsigned char)( 0 + c)] = '0' + c;
}
for (unsigned char c = 0; c < 6; c++) {
from_hex_lut[(unsigned char)('a' + c)] = 10 + c;
from_hex_lut[(unsigned char)('A' + c)] = 10 + c;
to_hex_lut[ (unsigned char)(10 + c)] = 'a' + c;
}
}
constexpr char from_hex(unsigned char c) const noexcept { return from_hex_lut[c]; }
constexpr char to_hex(unsigned char b) const noexcept { return to_hex_lut[b]; }
} constexpr hex_lut;
// This main point of this static assert is to force the compiler to compile-time build the constexpr tables.
static_assert(hex_lut.from_hex('a') == 10 && hex_lut.from_hex('F') == 15 && hex_lut.to_hex(13) == 'd', "");
} // namespace detail
/// Returns the number of characters required to encode a hex string from the given number of bytes.
inline constexpr size_t to_hex_size(size_t byte_size) { return byte_size * 2; }
/// Returns the number of bytes required to decode a hex string of the given size.
inline constexpr size_t from_hex_size(size_t hex_size) { return hex_size / 2; }
/// Iterable object for on-the-fly hex encoding. Used internally, but also particularly useful when
/// converting from one encoding to another.
template <typename InputIt>
struct hex_encoder final {
private:
InputIt _it, _end;
static_assert(sizeof(decltype(*_it)) == 1, "hex_encoder requires chars/bytes input iterator");
uint8_t c = 0;
bool second_half = false;
public:
using iterator_category = std::input_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = char;
using reference = value_type;
using pointer = void;
hex_encoder(InputIt begin, InputIt end) : _it{std::move(begin)}, _end{std::move(end)} {}
hex_encoder end() { return {_end, _end}; }
bool operator==(const hex_encoder& i) { return _it == i._it && second_half == i.second_half; }
bool operator!=(const hex_encoder& i) { return !(*this == i); }
hex_encoder& operator++() {
second_half = !second_half;
if (!second_half)
++_it;
return *this;
}
hex_encoder operator++(int) { hex_encoder copy{*this}; ++*this; return copy; }
char operator*() {
return detail::hex_lut.to_hex(second_half
? c & 0x0f
: (c = static_cast<uint8_t>(*_it)) >> 4);
}
};
/// Creates hex digits from a character sequence given by iterators, writes them starting at `out`.
/// Returns the final value of out (i.e. the iterator positioned just after the last written
/// hex character).
template <typename InputIt, typename OutputIt>
OutputIt to_hex(InputIt begin, InputIt end, OutputIt out) {
static_assert(sizeof(decltype(*begin)) == 1, "to_hex requires chars/bytes");
auto it = hex_encoder{begin, end};
return std::copy(it, it.end(), out);
}
/// Creates a string of hex digits from a character sequence iterator pair
template <typename It>
std::string to_hex(It begin, It end) {
std::string hex;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>) {
using std::distance;
hex.reserve(to_hex_size(distance(begin, end)));
}
to_hex(begin, end, std::back_inserter(hex));
return hex;
}
/// Creates a hex string from an iterable, std::string-like object
template <typename CharT>
std::string to_hex(std::basic_string_view<CharT> s) { return to_hex(s.begin(), s.end()); }
inline std::string to_hex(std::string_view s) { return to_hex<>(s); }
/// Returns true if the given value is a valid hex digit.
template <typename CharT>
constexpr bool is_hex_digit(CharT c) {
static_assert(sizeof(CharT) == 1, "is_hex requires chars/bytes");
return detail::hex_lut.from_hex(static_cast<unsigned char>(c)) != 0 || static_cast<unsigned char>(c) == '0';
}
/// Returns true if all elements in the range are hex characters *and* the string length is a
/// multiple of 2, and thus suitable to pass to from_hex().
template <typename It>
constexpr bool is_hex(It begin, It end) {
static_assert(sizeof(decltype(*begin)) == 1, "is_hex requires chars/bytes");
constexpr bool ra = std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>;
if constexpr (ra) {
using std::distance;
if (distance(begin, end) % 2 != 0)
return false;
}
size_t count = 0;
for (; begin != end; ++begin) {
if constexpr (!ra) ++count;
if (!is_hex_digit(*begin))
return false;
}
if constexpr (!ra)
return count % 2 == 0;
return true;
}
/// Returns true if all elements in the string-like value are hex characters
template <typename CharT>
constexpr bool is_hex(std::basic_string_view<CharT> s) { return is_hex(s.begin(), s.end()); }
constexpr bool is_hex(std::string_view s) { return is_hex(s.begin(), s.end()); }
/// Convert a hex digit into its numeric (0-15) value
constexpr char from_hex_digit(unsigned char x) noexcept {
return detail::hex_lut.from_hex(x);
}
/// Constructs a byte value from a pair of hex digits
constexpr char from_hex_pair(unsigned char a, unsigned char b) noexcept { return (from_hex_digit(a) << 4) | from_hex_digit(b); }
/// Iterable object for on-the-fly hex decoding. Used internally but also particularly useful when
/// converting from one encoding to another. Undefined behaviour if the given iterator range is not
/// a valid hex string with even length (i.e. is_hex() should return true).
template <typename InputIt>
struct hex_decoder final {
private:
InputIt _it, _end;
static_assert(sizeof(decltype(*_it)) == 1, "hex_encoder requires chars/bytes input iterator");
char byte;
public:
using iterator_category = std::input_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = char;
using reference = value_type;
using pointer = void;
hex_decoder(InputIt begin, InputIt end) : _it{std::move(begin)}, _end{std::move(end)} {
if (_it != _end)
load_byte();
}
hex_decoder end() { return {_end, _end}; }
bool operator==(const hex_decoder& i) { return _it == i._it; }
bool operator!=(const hex_decoder& i) { return _it != i._it; }
hex_decoder& operator++() {
if (++_it != _end)
load_byte();
return *this;
}
hex_decoder operator++(int) { hex_decoder copy{*this}; ++*this; return copy; }
char operator*() const { return byte; }
private:
void load_byte() {
auto a = *_it;
auto b = *++_it;
byte = from_hex_pair(static_cast<unsigned char>(a), static_cast<unsigned char>(b));
}
};
/// Converts a sequence of hex digits to bytes. Undefined behaviour if any characters are not in
/// [0-9a-fA-F] or if the input sequence length is not even: call `is_hex` first if you need to
/// check. It is permitted for the input and output ranges to overlap as long as out is no later
/// than begin. Returns the final value of out (that is, the iterator positioned just after the
/// last written character).
template <typename InputIt, typename OutputIt>
OutputIt from_hex(InputIt begin, InputIt end, OutputIt out) {
assert(is_hex(begin, end));
auto it = hex_decoder(begin, end);
const auto hend = it.end();
while (it != hend)
*out++ = static_cast<detail::byte_type_t<OutputIt>>(*it++);
return out;
}
/// Converts a sequence of hex digits to a string of bytes and returns it. Undefined behaviour if
/// the input sequence is not an even-length sequence of [0-9a-fA-F] characters.
template <typename It>
std::string from_hex(It begin, It end) {
std::string bytes;
if constexpr (std::is_base_of_v<std::random_access_iterator_tag, typename std::iterator_traits<It>::iterator_category>) {
using std::distance;
bytes.reserve(from_hex_size(distance(begin, end)));
}
from_hex(begin, end, std::back_inserter(bytes));
return bytes;
}
/// Converts hex digits from a std::string-like object into a std::string of bytes. Undefined
/// behaviour if any characters are not in [0-9a-fA-F] or if the input sequence length is not even.
template <typename CharT>
std::string from_hex(std::basic_string_view<CharT> s) { return from_hex(s.begin(), s.end()); }
inline std::string from_hex(std::string_view s) { return from_hex<>(s); }
using oxenc::to_hex_size;
using oxenc::from_hex_size;
using oxenc::hex_encoder;
using oxenc::to_hex;
using oxenc::is_hex_digit;
using oxenc::is_hex;
using oxenc::hex_decoder;
using oxenc::from_hex_digit;
using oxenc::from_hex_pair;
using oxenc::from_hex;
}

8
oxenmq/jobs.cpp
View File

@ -32,7 +32,7 @@ void OxenMQ::job(std::function<void()> f, std::optional<TaggedThreadID> thread)
auto* b = new Batch<void>;
b->add_job(std::move(f), thread);
auto* baseptr = static_cast<detail::Batch*>(b);
detail::send_control(get_control_socket(), "BATCH", bt_serialize(reinterpret_cast<uintptr_t>(baseptr)));
detail::send_control(get_control_socket(), "BATCH", oxenc::bt_serialize(reinterpret_cast<uintptr_t>(baseptr)));
}
void OxenMQ::proxy_schedule_reply_job(std::function<void()> f) {
@ -68,7 +68,7 @@ void OxenMQ::proxy_timer(int id, std::function<void()> job, std::chrono::millise
timer_zmq_id[id] = zmq_timer_id;
}
void OxenMQ::proxy_timer(bt_list_consumer timer_data) {
void OxenMQ::proxy_timer(oxenc::bt_list_consumer timer_data) {
auto timer_id = timer_data.consume_integer<int>();
std::unique_ptr<std::function<void()>> func{reinterpret_cast<std::function<void()>*>(timer_data.consume_integer<uintptr_t>())};
auto interval = std::chrono::milliseconds{timer_data.consume_integer<uint64_t>()};
@ -124,7 +124,7 @@ void OxenMQ::add_timer(TimerID& timer, std::function<void()> job, std::chrono::m
int th_id = thread ? thread->_id : 0;
timer._id = next_timer_id++;
if (proxy_thread.joinable()) {
detail::send_control(get_control_socket(), "TIMER", bt_serialize(bt_list{{
detail::send_control(get_control_socket(), "TIMER", oxenc::bt_serialize(oxenc::bt_list{{
timer._id,
detail::serialize_object(std::move(job)),
interval.count(),
@ -153,7 +153,7 @@ void OxenMQ::proxy_timer_del(int id) {
void OxenMQ::cancel_timer(TimerID timer_id) {
if (proxy_thread.joinable()) {
detail::send_control(get_control_socket(), "TIMER_DEL", bt_serialize(timer_id._id));
detail::send_control(get_control_socket(), "TIMER_DEL", oxenc::bt_serialize(timer_id._id));
} else {
proxy_timer_del(timer_id._id);
}

2
oxenmq/oxenmq-internal.h
View File

@ -115,7 +115,7 @@ inline AuthLevel auth_from_string(std::string_view a) {
}
// Extracts and builds the "send" part of a message for proxy_send/proxy_reply
inline std::list<zmq::message_t> build_send_parts(bt_list_consumer send, std::string_view route) {
inline std::list<zmq::message_t> build_send_parts(oxenc::bt_list_consumer send, std::string_view route) {
std::list<zmq::message_t> parts;
if (!route.empty())
parts.push_back(create_message(route));

15
oxenmq/oxenmq.cpp
View File

@ -12,7 +12,8 @@ extern "C" {
#include <sodium/crypto_box.h>
#include <sodium/crypto_scalarmult.h>
}
#include "hex.h"
#include <oxenc/hex.h>
#include <oxenc/variant.h>
namespace oxenmq {
@ -21,7 +22,7 @@ namespace {
/// Creates a message by bt-serializing the given value (string, number, list, or dict)
template <typename T>
zmq::message_t create_bt_message(T&& data) { return create_message(bt_serialize(std::forward<T>(data))); }
zmq::message_t create_bt_message(T&& data) { return create_message(oxenc::bt_serialize(std::forward<T>(data))); }
template <typename MessageContainer>
std::vector<std::string> as_strings(const MessageContainer& msgs) {
@ -62,9 +63,9 @@ std::pair<std::string, AuthLevel> extract_metadata(zmq::message_t& msg) {
std::string_view pubkey_hex{msg.gets("User-Id")};
if (pubkey_hex.size() != 64)
throw std::logic_error("bad user-id");
assert(is_hex(pubkey_hex.begin(), pubkey_hex.end()));
assert(oxenc::is_hex(pubkey_hex.begin(), pubkey_hex.end()));
result.first.resize(32, 0);
from_hex(pubkey_hex.begin(), pubkey_hex.end(), result.first.begin());
oxenc::from_hex(pubkey_hex.begin(), pubkey_hex.end(), result.first.begin());
} catch (...) {}
try {
@ -233,7 +234,7 @@ void OxenMQ::start() {
if (proxy_thread.joinable())
throw std::logic_error("Cannot call start() multiple times!");
OMQ_LOG(info, "Initializing OxenMQ ", bind.empty() ? "remote-only" : "listener", " with pubkey ", to_hex(pubkey));
OMQ_LOG(info, "Initializing OxenMQ ", bind.empty() ? "remote-only" : "listener", " with pubkey ", oxenc::to_hex(pubkey));
int zmq_socket_limit = context.get(zmq::ctxopt::socket_limit);
if (MAX_SOCKETS > 1 && MAX_SOCKETS <= zmq_socket_limit)
@ -273,7 +274,7 @@ void OxenMQ::listen_curve(std::string bind_addr, AllowFunc allow_connection, std
if (!allow_connection) allow_connection = [](auto&&...) { return AuthLevel::none; };
bind_data d{std::move(bind_addr), true, std::move(allow_connection), std::move(on_bind)};
if (proxy_thread.joinable())
detail::send_control(get_control_socket(), "BIND", bt_serialize(detail::serialize_object(std::move(d))));
detail::send_control(get_control_socket(), "BIND", oxenc::bt_serialize(detail::serialize_object(std::move(d))));
else
bind.push_back(std::move(d));
}
@ -284,7 +285,7 @@ void OxenMQ::listen_plain(std::string bind_addr, AllowFunc allow_connection, std
if (!allow_connection) allow_connection = [](auto&&...) { return AuthLevel::none; };
bind_data d{std::move(bind_addr), false, std::move(allow_connection), std::move(on_bind)};
if (proxy_thread.joinable())
detail::send_control(get_control_socket(), "BIND", bt_serialize(detail::serialize_object(std::move(d))));
detail::send_control(get_control_socket(), "BIND", oxenc::bt_serialize(detail::serialize_object(std::move(d))));
else
bind.push_back(std::move(d));
}

76
oxenmq/oxenmq.h
View File

@ -47,7 +47,7 @@
#include <future>
#include "zmq.hpp"
#include "address.h"
#include "bt_serialize.h"
#include <oxenc/bt_serialize.h>
#include "connections.h"
#include "message.h"
#include "auth.h"
@ -558,22 +558,22 @@ private:
/// CONNECT_SN command telling us to connect to a new pubkey. Returns the socket (which could
/// be existing or a new one). This basically just unpacks arguments and passes them on to
/// proxy_connect_sn().
std::pair<zmq::socket_t*, std::string> proxy_connect_sn(bt_dict_consumer data);
std::pair<zmq::socket_t*, std::string> proxy_connect_sn(oxenc::bt_dict_consumer data);
/// Opens a new connection to a remote, with callbacks. This is the proxy-side implementation
/// of the `connect_remote()` call.
void proxy_connect_remote(bt_dict_consumer data);
void proxy_connect_remote(oxenc::bt_dict_consumer data);
/// Called to disconnect our remote connection to the given id (if we have one).
void proxy_disconnect(bt_dict_consumer data);
void proxy_disconnect(oxenc::bt_dict_consumer data);
void proxy_disconnect(ConnectionID conn, std::chrono::milliseconds linger);
/// SEND command. Does a connect first, if necessary.
void proxy_send(bt_dict_consumer data);
void proxy_send(oxenc::bt_dict_consumer data);
/// REPLY command. Like SEND, but only has a listening socket route to send back to and so is
/// weaker (i.e. it cannot reconnect to the SN if the connection is no longer open).
void proxy_reply(bt_dict_consumer data);
void proxy_reply(oxenc::bt_dict_consumer data);
/// Currently active batch/reply jobs; this is the container that owns the Batch instances
std::unordered_set<detail::Batch*> batches;
@ -595,7 +595,7 @@ private:
/// TIMER command. Called with a serialized list containing: our local timer_id, function
/// pointer to assume ownership of, an interval count (in ms), and whether or not jobs should be
/// squelched (see `add_timer()`).
void proxy_timer(bt_list_consumer timer_data);
void proxy_timer(oxenc::bt_list_consumer timer_data);
/// Same, but deserialized
void proxy_timer(int timer_id, std::function<void()> job, std::chrono::milliseconds interval, bool squelch, int thread);
@ -671,7 +671,7 @@ private:
/// Resets or updates the stored set of active SN pubkeys
void proxy_set_active_sns(std::string_view data);
void proxy_set_active_sns(pubkey_set pubkeys);
void proxy_update_active_sns(bt_list_consumer data);
void proxy_update_active_sns(oxenc::bt_list_consumer data);
void proxy_update_active_sns(pubkey_set added, pubkey_set removed);
void proxy_update_active_sns_clean(pubkey_set added, pubkey_set removed);
@ -1563,64 +1563,64 @@ template <typename T> T deserialize_object(uintptr_t ptrval) {
void send_control(zmq::socket_t& sock, std::string_view cmd, std::string data = {});
/// Base case: takes a string-like value and appends it to the message parts
inline void apply_send_option(bt_list& parts, bt_dict&, std::string_view arg) {
inline void apply_send_option(oxenc::bt_list& parts, oxenc::bt_dict&, std::string_view arg) {
parts.emplace_back(arg);
}
/// std::optional<T>: if the optional is set, we unwrap it and apply as a send_option, otherwise we
/// ignore it.
template <typename T>
inline void apply_send_option(bt_list& parts, bt_dict& control_data, const std::optional<T>& opt) {
inline void apply_send_option(oxenc::bt_list& parts, oxenc::bt_dict& control_data, const std::optional<T>& opt) {
if (opt) apply_send_option(parts, control_data, *opt);
}
/// `data_parts` specialization: appends a range of serialized data parts to the parts to send
template <typename InputIt>
void apply_send_option(bt_list& parts, bt_dict&, const send_option::data_parts_impl<InputIt> data) {
void apply_send_option(oxenc::bt_list& parts, oxenc::bt_dict&, const send_option::data_parts_impl<InputIt> data) {
for (auto it = data.begin; it != data.end; ++it)
parts.emplace_back(*it);
}
/// `hint` specialization: sets the hint in the control data
inline void apply_send_option(bt_list&, bt_dict& control_data, const send_option::hint& hint) {
inline void apply_send_option(oxenc::bt_list&, oxenc::bt_dict& control_data, const send_option::hint& hint) {
if (hint.connect_hint.empty()) return;
control_data["hint"] = hint.connect_hint;
}
/// `optional` specialization: sets the optional flag in the control data
inline void apply_send_option(bt_list&, bt_dict& control_data, const send_option::optional& o) {
inline void apply_send_option(oxenc::bt_list&, oxenc::bt_dict& control_data, const send_option::optional& o) {
control_data["optional"] = o.is_optional;
}
/// `incoming` specialization: sets the incoming-only flag in the control data
inline void apply_send_option(bt_list&, bt_dict& control_data, const send_option::incoming& i) {
inline void apply_send_option(oxenc::bt_list&, oxenc::bt_dict& control_data, const send_option::incoming& i) {
control_data["incoming"] = i.is_incoming;
}
/// `outgoing` specialization: sets the outgoing-only flag in the control data
inline void apply_send_option(bt_list&, bt_dict& control_data, const send_option::outgoing& o) {
inline void apply_send_option(oxenc::bt_list&, oxenc::bt_dict& control_data, const send_option::outgoing& o) {
control_data["outgoing"] = o.is_outgoing;
}
/// `keep_alive` specialization: increases the outgoing socket idle timeout (if shorter)
inline void apply_send_option(bt_list&, bt_dict& control_data, const send_option::keep_alive& timeout) {
inline void apply_send_option(oxenc::bt_list&, oxenc::bt_dict& control_data, const send_option::keep_alive& timeout) {
if (timeout.time >= 0ms)
control_data["keep_alive"] = timeout.time.count();
}
/// `request_timeout` specialization: set the timeout time for a request
inline void apply_send_option(bt_list&, bt_dict& control_data, const send_option::request_timeout& timeout) {
inline void apply_send_option(oxenc::bt_list&, oxenc::bt_dict& control_data, const send_option::request_timeout& timeout) {
if (timeout.time >= 0ms)
control_data["request_timeout"] = timeout.time.count();
}
/// `queue_failure` specialization
inline void apply_send_option(bt_list&, bt_dict& control_data, send_option::queue_failure f) {
inline void apply_send_option(oxenc::bt_list&, oxenc::bt_dict& control_data, send_option::queue_failure f) {
control_data["send_fail"] = serialize_object(std::move(f.callback));
}
/// `queue_full` specialization
inline void apply_send_option(bt_list&, bt_dict& control_data, send_option::queue_full f) {
inline void apply_send_option(oxenc::bt_list&, oxenc::bt_dict& control_data, send_option::queue_full f) {
control_data["send_full_q"] = serialize_object(std::move(f.callback));
}
@ -1628,9 +1628,9 @@ inline void apply_send_option(bt_list&, bt_dict& control_data, send_option::queu
std::pair<std::string, AuthLevel> extract_metadata(zmq::message_t& msg);
template <typename... T>
bt_dict build_send(ConnectionID to, std::string_view cmd, T&&... opts) {
bt_dict control_data;
bt_list parts{{cmd}};
oxenc::bt_dict build_send(ConnectionID to, std::string_view cmd, T&&... opts) {
oxenc::bt_dict control_data;
oxenc::bt_list parts{{cmd}};
(detail::apply_send_option(parts, control_data, std::forward<T>(opts)),...);
if (to.sn())
@ -1644,34 +1644,34 @@ bt_dict build_send(ConnectionID to, std::string_view cmd, T&&... opts) {
}
inline void apply_connect_option(OxenMQ& omq, bool remote, bt_dict& opts, const AuthLevel& auth) {
inline void apply_connect_option(OxenMQ& omq, bool remote, oxenc::bt_dict& opts, const AuthLevel& auth) {
if (remote) opts["auth_level"] = static_cast<std::underlying_type_t<AuthLevel>>(auth);
else omq.log(LogLevel::warn, __FILE__, __LINE__, "AuthLevel ignored for connect_sn(...)");
}
inline void apply_connect_option(OxenMQ&, bool, bt_dict& opts, const connect_option::ephemeral_routing_id& er) {
inline void apply_connect_option(OxenMQ&, bool, oxenc::bt_dict& opts, const connect_option::ephemeral_routing_id& er) {
opts["ephemeral_rid"] = er.use_ephemeral_routing_id;
}
inline void apply_connect_option(OxenMQ& omq, bool remote, bt_dict& opts, const connect_option::timeout& timeout) {
inline void apply_connect_option(OxenMQ& omq, bool remote, oxenc::bt_dict& opts, const connect_option::timeout& timeout) {
if (remote) opts["timeout"] = timeout.time.count();
else omq.log(LogLevel::warn, __FILE__, __LINE__, "connect_option::timeout ignored for connect_sn(...)");
}
inline void apply_connect_option(OxenMQ& omq, bool remote, bt_dict& opts, const connect_option::keep_alive& ka) {
inline void apply_connect_option(OxenMQ& omq, bool remote, oxenc::bt_dict& opts, const connect_option::keep_alive& ka) {
if (ka.time < 0ms) return;
else if (!remote) opts["keep_alive"] = ka.time.count();
else omq.log(LogLevel::warn, __FILE__, __LINE__, "connect_option::keep_alive ignored for connect_remote(...)");
}
inline void apply_connect_option(OxenMQ& omq, bool remote, bt_dict& opts, const connect_option::hint& hint) {
inline void apply_connect_option(OxenMQ& omq, bool remote, oxenc::bt_dict& opts, const connect_option::hint& hint) {
if (hint.address.empty()) return;
if (!remote) opts["hint"] = hint.address;
else omq.log(LogLevel::warn, __FILE__, __LINE__, "connect_option::hint ignored for connect_remote(...)");
}
[[deprecated("use oxenmq::connect_option::keep_alive or ::timeout instead")]]
inline void apply_connect_option(OxenMQ&, bool remote, bt_dict& opts, std::chrono::milliseconds time) {
inline void apply_connect_option(OxenMQ&, bool remote, oxenc::bt_dict& opts, std::chrono::milliseconds time) {
if (remote) opts["timeout"] = time.count();
else opts["keep_alive"] = time.count();
}
[[deprecated("use oxenmq::connect_option::hint{hint} instead of a direct string argument")]]
inline void apply_connect_option(OxenMQ& omq, bool remote, bt_dict& opts, std::string_view hint) {
inline void apply_connect_option(OxenMQ& omq, bool remote, oxenc::bt_dict& opts, std::string_view hint) {
if (!remote) opts["hint"] = hint;
else omq.log(LogLevel::warn, __FILE__, __LINE__, "string argument ignored for connect_remote(...)");
}
@ -1681,7 +1681,7 @@ inline void apply_connect_option(OxenMQ& omq, bool remote, bt_dict& opts, std::s
template <typename... Option>
ConnectionID OxenMQ::connect_remote(const address& remote, ConnectSuccess on_connect, ConnectFailure on_failure,
const Option&... options) {
bt_dict opts;
oxenc::bt_dict opts;
(detail::apply_connect_option(*this, true, opts, options), ...);
auto id = next_conn_id++;
@ -1691,14 +1691,14 @@ ConnectionID OxenMQ::connect_remote(const address& remote, ConnectSuccess on_con
if (remote.curve()) opts["pubkey"] = remote.pubkey;
opts["remote"] = remote.zmq_address();
detail::send_control(get_control_socket(), "CONNECT_REMOTE", bt_serialize(opts));
detail::send_control(get_control_socket(), "CONNECT_REMOTE", oxenc::bt_serialize(opts));
return id;
}
template <typename... Option>
ConnectionID OxenMQ::connect_sn(std::string_view pubkey, const Option&... options) {
bt_dict opts{
oxenc::bt_dict opts{
{"keep_alive", std::chrono::microseconds{DEFAULT_CONNECT_SN_KEEP_ALIVE}.count()},
{"ephemeral_rid", EPHEMERAL_ROUTING_ID},
};
@ -1707,7 +1707,7 @@ ConnectionID OxenMQ::connect_sn(std::string_view pubkey, const Option&... option
opts["pubkey"] = pubkey;
detail::send_control(get_control_socket(), "CONNECT_SN", bt_serialize(opts));
detail::send_control(get_control_socket(), "CONNECT_SN", oxenc::bt_serialize(opts));
return pubkey;
}
@ -1715,7 +1715,7 @@ ConnectionID OxenMQ::connect_sn(std::string_view pubkey, const Option&... option
template <typename... Option>
ConnectionID OxenMQ::connect_inproc(ConnectSuccess on_connect, ConnectFailure on_failure,
const Option&... options) {
bt_dict opts{
oxenc::bt_dict opts{
{"timeout", INPROC_CONNECT_TIMEOUT.count()},
{"auth_level", static_cast<std::underlying_type_t<AuthLevel>>(AuthLevel::admin)}
};
@ -1728,7 +1728,7 @@ ConnectionID OxenMQ::connect_inproc(ConnectSuccess on_connect, ConnectFailure on
opts["failure"] = detail::serialize_object(std::move(on_failure));
opts["remote"] = "inproc://sn-self";
detail::send_control(get_control_socket(), "CONNECT_REMOTE", bt_serialize(opts));
detail::send_control(get_control_socket(), "CONNECT_REMOTE", oxenc::bt_serialize(opts));
return id;
}
@ -1736,7 +1736,7 @@ ConnectionID OxenMQ::connect_inproc(ConnectSuccess on_connect, ConnectFailure on
template <typename... T>
void OxenMQ::send(ConnectionID to, std::string_view cmd, const T&... opts) {
detail::send_control(get_control_socket(), "SEND",
bt_serialize(detail::build_send(std::move(to), cmd, opts...)));
oxenc::bt_serialize(detail::build_send(std::move(to), cmd, opts...)));
}
std::string make_random_string(size_t size);
@ -1744,11 +1744,11 @@ std::string make_random_string(size_t size);
template <typename... T>
void OxenMQ::request(ConnectionID to, std::string_view cmd, ReplyCallback callback, const T &...opts) {
const auto reply_tag = make_random_string(15); // 15 random bytes is lots and should keep us in most stl implementations' small string optimization
bt_dict control_data = detail::build_send(std::move(to), cmd, reply_tag, opts...);
oxenc::bt_dict control_data = detail::build_send(std::move(to), cmd, reply_tag, opts...);
control_data["request"] = true;
control_data["request_callback"] = detail::serialize_object(std::move(callback));
control_data["request_tag"] = std::string_view{reply_tag};
detail::send_control(get_control_socket(), "SEND", bt_serialize(std::move(control_data)));
detail::send_control(get_control_socket(), "SEND", oxenc::bt_serialize(std::move(control_data)));
}
template <typename... Args>

34
oxenmq/proxy.cpp
View File

@ -1,6 +1,6 @@
#include "oxenmq.h"
#include "oxenmq-internal.h"
#include "hex.h"
#include <oxenc/hex.h>
#include <exception>
#include <future>
@ -43,7 +43,7 @@ void OxenMQ::proxy_quit() {
OMQ_LOG(debug, "Proxy thread teardown complete");
}
void OxenMQ::proxy_send(bt_dict_consumer data) {
void OxenMQ::proxy_send(oxenc::bt_dict_consumer data) {
// NB: bt_dict_consumer goes in alphabetical order
std::string_view hint;
std::chrono::milliseconds keep_alive{DEFAULT_SEND_KEEP_ALIVE};
@ -99,7 +99,7 @@ void OxenMQ::proxy_send(bt_dict_consumer data) {
}
if (!data.skip_until("send"))
throw std::runtime_error("Internal error: Invalid proxy send command; send parts missing");
bt_list_consumer send = data.consume_list_consumer();
oxenc::bt_list_consumer send = data.consume_list_consumer();
send_option::queue_failure::callback_t callback_nosend;
if (data.skip_until("send_fail"))
@ -123,9 +123,9 @@ void OxenMQ::proxy_send(bt_dict_consumer data) {
nowarn = true;
if (optional)
OMQ_LOG(debug, "Not sending: send is optional and no connection to ",
to_hex(conn_id.pk), " is currently established");
oxenc::to_hex(conn_id.pk), " is currently established");
else
OMQ_LOG(error, "Unable to send to ", to_hex(conn_id.pk), ": no valid connection address found");
OMQ_LOG(error, "Unable to send to ", oxenc::to_hex(conn_id.pk), ": no valid connection address found");
break;
}
send_to = sock_route.first;
@ -176,7 +176,7 @@ void OxenMQ::proxy_send(bt_dict_consumer data) {
// The incoming connection to the SN is no longer good, but we can retry because
// we may have another active connection with the SN (or may want to open one).
if (removed) {
OMQ_LOG(debug, "Retrying sending to SN ", to_hex(conn_id.pk), " using other sockets");
OMQ_LOG(debug, "Retrying sending to SN ", oxenc::to_hex(conn_id.pk), " using other sockets");
retry = true;
}
}
@ -199,7 +199,7 @@ void OxenMQ::proxy_send(bt_dict_consumer data) {
}
if (request) {
if (sent) {
OMQ_LOG(debug, "Added new pending request ", to_hex(request_tag));
OMQ_LOG(debug, "Added new pending request ", oxenc::to_hex(request_tag));
pending_requests.insert({ request_tag, {
std::chrono::steady_clock::now() + request_timeout, std::move(request_callback) }});
} else {
@ -217,7 +217,7 @@ void OxenMQ::proxy_send(bt_dict_consumer data) {
}
}
void OxenMQ::proxy_reply(bt_dict_consumer data) {
void OxenMQ::proxy_reply(oxenc::bt_dict_consumer data) {
bool have_conn_id = false;
ConnectionID conn_id{0};
if (data.skip_until("conn_id")) {
@ -236,7 +236,7 @@ void OxenMQ::proxy_reply(bt_dict_consumer data) {
if (!data.skip_until("send"))
throw std::runtime_error("Internal error: Invalid proxy reply command; send parts missing");
bt_list_consumer send = data.consume_list_consumer();
oxenc::bt_list_consumer send = data.consume_list_consumer();
auto pr = peers.equal_range(conn_id);
if (pr.first == pr.second) {
@ -289,11 +289,11 @@ void OxenMQ::proxy_control_message(std::vector<zmq::message_t>& parts) {
return proxy_reply(data);
} else if (cmd == "BATCH") {
OMQ_TRACE("proxy batch jobs");
auto ptrval = bt_deserialize<uintptr_t>(data);
auto ptrval = oxenc::bt_deserialize<uintptr_t>(data);
return proxy_batch(reinterpret_cast<detail::Batch*>(ptrval));
} else if (cmd == "INJECT") {
OMQ_TRACE("proxy inject");
return proxy_inject_task(detail::deserialize_object<injected_task>(bt_deserialize<uintptr_t>(data)));
return proxy_inject_task(detail::deserialize_object<injected_task>(oxenc::bt_deserialize<uintptr_t>(data)));
} else if (cmd == "SET_SNS") {
return proxy_set_active_sns(data);
} else if (cmd == "UPDATE_SNS") {
@ -308,9 +308,9 @@ void OxenMQ::proxy_control_message(std::vector<zmq::message_t>& parts) {
} else if (cmd == "TIMER") {
return proxy_timer(data);
} else if (cmd == "TIMER_DEL") {
return proxy_timer_del(bt_deserialize<int>(data));
return proxy_timer_del(oxenc::bt_deserialize<int>(data));
} else if (cmd == "BIND") {
auto b = detail::deserialize_object<bind_data>(bt_deserialize<uintptr_t>(data));
auto b = detail::deserialize_object<bind_data>(oxenc::bt_deserialize<uintptr_t>(data));
if (proxy_bind(b, bind.size()))
bind.push_back(std::move(b));
return;
@ -626,7 +626,7 @@ bool OxenMQ::proxy_handle_builtin(int64_t conn_id, zmq::socket_t& sock, std::vec
std::string reply_tag{view(parts[tag_pos])};
auto it = pending_requests.find(reply_tag);
if (it != pending_requests.end()) {
OMQ_LOG(debug, "Received REPLY for pending command ", to_hex(reply_tag), "; scheduling callback");
OMQ_LOG(debug, "Received REPLY for pending command ", oxenc::to_hex(reply_tag), "; scheduling callback");
std::vector<std::string> data;
data.reserve(parts.size() - (tag_pos + 1));
for (auto it = parts.begin() + (tag_pos + 1); it != parts.end(); ++it)
@ -636,7 +636,7 @@ bool OxenMQ::proxy_handle_builtin(int64_t conn_id, zmq::socket_t& sock, std::vec
});
pending_requests.erase(it);
} else {
OMQ_LOG(warn, "Received REPLY with unknown or already handled reply tag (", to_hex(reply_tag), "); ignoring");
OMQ_LOG(warn, "Received REPLY with unknown or already handled reply tag (", oxenc::to_hex(reply_tag), "); ignoring");
}
return true;
} else if (cmd == "HI") {
@ -705,13 +705,13 @@ bool OxenMQ::proxy_handle_builtin(int64_t conn_id, zmq::socket_t& sock, std::vec
std::string reply_tag{view(parts[2 + incoming])};
auto it = pending_requests.find(reply_tag);
if (it != pending_requests.end()) {
OMQ_LOG(debug, "Received ", cmd, " REPLY for pending command ", to_hex(reply_tag), "; scheduling failure callback");
OMQ_LOG(debug, "Received ", cmd, " REPLY for pending command ", oxenc::to_hex(reply_tag), "; scheduling failure callback");
proxy_schedule_reply_job([callback=std::move(it->second.second), cmd=std::string{cmd}] {
callback(false, {{std::move(cmd)}});
});
pending_requests.erase(it);
} else {
OMQ_LOG(warn, "Received REPLY with unknown or already handled reply tag (", to_hex(reply_tag), "); ignoring");
OMQ_LOG(warn, "Received REPLY with unknown or already handled reply tag (", oxenc::to_hex(reply_tag), "); ignoring");
}
} else {
OMQ_LOG(warn, "Received ", cmd, ':', (parts.size() > 1 + incoming ? view(parts[1 + incoming]) : "(unknown command)"sv),

102
oxenmq/variant.h
View File

@ -1,103 +1,5 @@
#pragma once
// Workarounds for macos compatibility. On macOS we aren't allowed to touch anything in
// std::variant that could throw if compiling with a target <10.14 because Apple fails hard at
// properly updating their STL. Thus, if compiling in such a mode, we have to introduce
// workarounds.
//
// This header defines a `var` namespace with `var::get` and `var::visit` implementations. On
// everything except broken backwards macos, this is just an alias to `std`. On broken backwards
// macos, we provide implementations that throw std::runtime_error in failure cases since the
// std::bad_variant_access exception can't be touched.
//
// You also get a BROKEN_APPLE_VARIANT macro defined if targetting a problematic mac architecture.
#include <variant>
// Compatibility shim for oxenc includes
#ifdef __APPLE__
# include <AvailabilityVersions.h>
# if defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED < MAC_OS_X_VERSION_10_14
# define BROKEN_APPLE_VARIANT
# endif
#endif
#ifndef BROKEN_APPLE_VARIANT
namespace var = std; // Oh look, actual C++17 support
#else
// Oh look, apple.
namespace var {
// Apple won't let us use std::visit or std::get if targetting some version of macos earlier than
// 10.14 because Apple is awful about not updating their STL. So we have to provide our own, and
// then call these without `std::` -- on crappy macos we'll come here, on everything else we'll ADL
// to the std:: implementation.
template <typename T, typename... Types>
constexpr T& get(std::variant<Types...>& var) {
if (auto* v = std::get_if<T>(&var)) return *v;
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <typename T, typename... Types>
constexpr const T& get(const std::variant<Types...>& var) {
if (auto* v = std::get_if<T>(&var)) return *v;
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <typename T, typename... Types>
constexpr const T&& get(const std::variant<Types...>&& var) {
if (auto* v = std::get_if<T>(&var)) return std::move(*v);
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <typename T, typename... Types>
constexpr T&& get(std::variant<Types...>&& var) {
if (auto* v = std::get_if<T>(&var)) return std::move(*v);
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, typename... Types>
constexpr auto& get(std::variant<Types...>& var) {
if (auto* v = std::get_if<I>(&var)) return *v;
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, typename... Types>
constexpr const auto& get(const std::variant<Types...>& var) {
if (auto* v = std::get_if<I>(&var)) return *v;
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, typename... Types>
constexpr const auto&& get(const std::variant<Types...>&& var) {
if (auto* v = std::get_if<I>(&var)) return std::move(*v);
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, typename... Types>
constexpr auto&& get(std::variant<Types...>&& var) {
if (auto* v = std::get_if<I>(&var)) return std::move(*v);
throw std::runtime_error{"Bad variant access -- variant does not contain the requested type"};
}
template <size_t I, size_t... More, class Visitor, class Variant>
constexpr auto visit_helper(Visitor&& vis, Variant&& var) {
if (var.index() == I)
return vis(var::get<I>(std::forward<Variant>(var)));
else if constexpr (sizeof...(More) > 0)
return visit_helper<More...>(std::forward<Visitor>(vis), std::forward<Variant>(var));
else
throw std::runtime_error{"Bad visit -- variant is valueless"};
}
template <size_t... Is, class Visitor, class Variant>
constexpr auto visit_helper(Visitor&& vis, Variant&& var, std::index_sequence<Is...>) {
return visit_helper<Is...>(std::forward<Visitor>(vis), std::forward<Variant>(var));
}
// Only handle a single variant here because multi-variant invocation is notably harder (and we
// don't need it).
template <class Visitor, class Variant>
constexpr auto visit(Visitor&& vis, Variant&& var) {
return visit_helper(std::forward<Visitor>(vis), std::forward<Variant>(var),
std::make_index_sequence<std::variant_size_v<std::remove_reference_t<Variant>>>{});
}
} // namespace var
#endif
#include <oxenc/variant.h>

7
oxenmq/worker.cpp
View File

@ -8,6 +8,7 @@ extern "C" {
#include <pthread_np.h>
}
#endif
#include <oxenc/variant.h>
namespace oxenmq {
@ -134,7 +135,7 @@ void OxenMQ::worker_thread(unsigned int index, std::optional<std::string> tagged
callback(message);
}
}
catch (const bt_deserialize_invalid& e) {
catch (const oxenc::bt_deserialize_invalid& e) {
OMQ_LOG(warn, worker_id, " deserialization failed: ", e.what(), "; ignoring request");
}
#ifndef BROKEN_APPLE_VARIANT
@ -185,7 +186,7 @@ void OxenMQ::proxy_worker_message(std::vector<zmq::message_t>& parts) {
assert(route.size() >= 2 && (route[0] == 'w' || route[0] == 't') && route[1] >= '0' && route[1] <= '9');
bool tagged_worker = route[0] == 't';
std::string_view worker_id_str{&route[1], route.size()-1}; // Chop off the leading "w" (or "t")
unsigned int worker_id = detail::extract_unsigned(worker_id_str);
unsigned int worker_id = oxenc::detail::extract_unsigned(worker_id_str);
if (!worker_id_str.empty() /* didn't consume everything */ ||
(tagged_worker
? 0 == worker_id || worker_id > tagged_workers.size() // tagged worker ids are indexed from 1 to N (0 means untagged)
@ -391,7 +392,7 @@ void OxenMQ::inject_task(const std::string& category, std::string command, std::
auto it = categories.find(category);
if (it == categories.end())
throw std::out_of_range{"Invalid category `" + category + "': category does not exist"};
detail::send_control(get_control_socket(), "INJECT", bt_serialize(detail::serialize_object(
detail::send_control(get_control_socket(), "INJECT", oxenc::bt_serialize(detail::serialize_object(
injected_task{it->second, std::move(command), std::move(remote), std::move(callback)})));
}

2
tests/CMakeLists.txt
View File

@ -5,10 +5,8 @@ add_executable(tests
main.cpp
test_address.cpp
test_batch.cpp
test_bt.cpp
test_connect.cpp
test_commands.cpp
test_encoding.cpp
test_failures.cpp
test_inject.cpp
test_requests.cpp

322
tests/test_bt.cpp
View File

@ -1,322 +0,0 @@
#include "oxenmq/bt_serialize.h"
#include "oxenmq/bt_producer.h"
#include "common.h"
#include <map>
#include <set>
#include <limits>
TEST_CASE("bt basic value serialization", "[bt][serialization]") {
int x = 42;
std::string x_ = bt_serialize(x);
REQUIRE( bt_serialize(x) == "i42e" );
int64_t ibig = -8'000'000'000'000'000'000LL;
uint64_t ubig = 10'000'000'000'000'000'000ULL;
REQUIRE( bt_serialize(ibig) == "i-8000000000000000000e" );
REQUIRE( bt_serialize(std::numeric_limits<int64_t>::min()) == "i-9223372036854775808e" );
REQUIRE( bt_serialize(ubig) == "i10000000000000000000e" );
REQUIRE( bt_serialize(std::numeric_limits<uint64_t>::max()) == "i18446744073709551615e" );
std::unordered_map<std::string, int> m;
m["hi"] = 123;
m["omg"] = -7890;
m["bye"] = 456;
m["zap"] = 0;
// bt values are always sorted:
REQUIRE( bt_serialize(m) == "d3:byei456e2:hii123e3:omgi-7890e3:zapi0ee" );
// Dict-like list serializes as a dict (and get sorted, as above)
std::list<std::pair<std::string, std::string>> d{{
{"c", "x"},
{"a", "z"},
{"b", "y"},
}};
REQUIRE( bt_serialize(d) == "d1:a1:z1:b1:y1:c1:xe" );
std::vector<std::string> v{{"a", "", "\x00"s, "\x00\x00\x00goo"s}};
REQUIRE( bt_serialize(v) == "l1:a0:1:\0006:\x00\x00\x00gooe"sv );
std::array v2 = {"a"sv, ""sv, "\x00"sv, "\x00\x00\x00goo"sv};
REQUIRE( bt_serialize(v2) == "l1:a0:1:\0006:\x00\x00\x00gooe"sv );
}
TEST_CASE("bt nested value serialization", "[bt][serialization]") {
std::unordered_map<std::string, std::list<std::map<std::string, std::set<int>>>> x{{
{"foo", {{{"a", {1,2,3}}, {"b", {}}}, {{"c", {4,-5}}}}},
{"bar", {}}
}};
REQUIRE( bt_serialize(x) == "d3:barle3:foold1:ali1ei2ei3ee1:bleed1:cli-5ei4eeeee" );
}
TEST_CASE("bt basic value deserialization", "[bt][deserialization]") {
REQUIRE( bt_deserialize<int>("i42e") == 42 );
int64_t ibig = -8'000'000'000'000'000'000LL;
uint64_t ubig = 10'000'000'000'000'000'000ULL;
REQUIRE( bt_deserialize<int64_t>("i-8000000000000000000e") == ibig );
REQUIRE( bt_deserialize<uint64_t>("i10000000000000000000e") == ubig );
REQUIRE( bt_deserialize<int64_t>("i-9223372036854775808e") == std::numeric_limits<int64_t>::min() );
REQUIRE( bt_deserialize<uint64_t>("i18446744073709551615e") == std::numeric_limits<uint64_t>::max() );
REQUIRE( bt_deserialize<uint32_t>("i4294967295e") == std::numeric_limits<uint32_t>::max() );
REQUIRE_THROWS( bt_deserialize<int64_t>("i-9223372036854775809e") );
REQUIRE_THROWS( bt_deserialize<uint64_t>("i-1e") );
REQUIRE_THROWS( bt_deserialize<uint32_t>("i4294967296e") );
std::unordered_map<std::string, int> m;
m["hi"] = 123;
m["omg"] = -7890;
m["bye"] = 456;
m["zap"] = 0;
// bt values are always sorted:
REQUIRE( bt_deserialize<std::unordered_map<std::string, int>>("d3:byei456e2:hii123e3:omgi-7890e3:zapi0ee") == m );
// Dict-like list can be used for deserialization
std::list<std::pair<std::string, std::string>> d{{
{"a", "z"},
{"b", "y"},
{"c", "x"},
}};
REQUIRE( bt_deserialize<std::list<std::pair<std::string, std::string>>>("d1:a1:z1:b1:y1:c1:xe") == d );
std::vector<std::string> v{{"a", "", "\x00"s, "\x00\x00\x00goo"s}};
REQUIRE( bt_deserialize<std::vector<std::string>>("l1:a0:1:\0006:\x00\x00\x00gooe"sv) == v );
std::vector v2 = {"a"sv, ""sv, "\x00"sv, "\x00\x00\x00goo"sv};
REQUIRE( bt_deserialize<decltype(v2)>("l1:a0:1:\0006:\x00\x00\x00gooe"sv) == v2 );
}
TEST_CASE("bt_value serialization", "[bt][serialization][bt_value]") {
bt_value dna{42};
std::string x_ = bt_serialize(dna);
REQUIRE( bt_serialize(dna) == "i42e" );
bt_value foo{"foo"};
REQUIRE( bt_serialize(foo) == "3:foo" );
bt_value ibig{-8'000'000'000'000'000'000LL};
bt_value ubig{10'000'000'000'000'000'000ULL};
int16_t ismall = -123;
uint16_t usmall = 123;
bt_dict nums{
{"a", 0},
{"b", -8'000'000'000'000'000'000LL},
{"c", 10'000'000'000'000'000'000ULL},
{"d", ismall},
{"e", usmall},
};
REQUIRE( bt_serialize(ibig) == "i-8000000000000000000e" );
REQUIRE( bt_serialize(ubig) == "i10000000000000000000e" );
REQUIRE( bt_serialize(nums) == "d1:ai0e1:bi-8000000000000000000e1:ci10000000000000000000e1:di-123e1:ei123ee" );
// Same as nested test, above, but with bt_* types
bt_dict x{{
{"foo", bt_list{{bt_dict{{ {"a", bt_list{{1,2,3}}}, {"b", bt_list{}}}}, bt_dict{{{"c", bt_list{{-5, 4}}}}}}}},
{"bar", bt_list{}}
}};
REQUIRE( bt_serialize(x) == "d3:barle3:foold1:ali1ei2ei3ee1:bleed1:cli-5ei4eeeee" );
std::vector<std::string> v{{"a", "", "\x00"s, "\x00\x00\x00goo"s}};
REQUIRE( bt_serialize(v) == "l1:a0:1:\0006:\x00\x00\x00gooe"sv );
std::array v2 = {"a"sv, ""sv, "\x00"sv, "\x00\x00\x00goo"sv};
REQUIRE( bt_serialize(v2) == "l1:a0:1:\0006:\x00\x00\x00gooe"sv );
}
TEST_CASE("bt_value deserialization", "[bt][deserialization][bt_value]") {
auto dna1 = bt_deserialize<bt_value>("i42e");
auto dna2 = bt_deserialize<bt_value>("i-42e");
REQUIRE( var::get<uint64_t>(dna1) == 42 );
REQUIRE( var::get<int64_t>(dna2) == -42 );
REQUIRE_THROWS( var::get<int64_t>(dna1) );
REQUIRE_THROWS( var::get<uint64_t>(dna2) );
REQUIRE( oxenmq::get_int<int>(dna1) == 42 );
REQUIRE( oxenmq::get_int<int>(dna2) == -42 );
REQUIRE( oxenmq::get_int<unsigned>(dna1) == 42 );
REQUIRE_THROWS( oxenmq::get_int<unsigned>(dna2) );
bt_value x = bt_deserialize<bt_value>("d3:barle3:foold1:ali1ei2ei3ee1:bleed1:cli-5ei4eeeee");
REQUIRE( std::holds_alternative<bt_dict>(x) );
bt_dict& a = var::get<bt_dict>(x);
REQUIRE( a.count("bar") );
REQUIRE( a.count("foo") );
REQUIRE( a.size() == 2 );
bt_list& foo = var::get<bt_list>(a["foo"]);
REQUIRE( foo.size() == 2 );
bt_dict& foo1 = var::get<bt_dict>(foo.front());
bt_dict& foo2 = var::get<bt_dict>(foo.back());
REQUIRE( foo1.size() == 2 );
REQUIRE( foo2.size() == 1 );
bt_list& foo1a = var::get<bt_list>(foo1.at("a"));
bt_list& foo1b = var::get<bt_list>(foo1.at("b"));
bt_list& foo2c = var::get<bt_list>(foo2.at("c"));
std::list<int> foo1a_vals, foo1b_vals, foo2c_vals;
for (auto& v : foo1a) foo1a_vals.push_back(oxenmq::get_int<int>(v));
for (auto& v : foo1b) foo1b_vals.push_back(oxenmq::get_int<int>(v));
for (auto& v : foo2c) foo2c_vals.push_back(oxenmq::get_int<int>(v));
REQUIRE( foo1a_vals == std::list{{1,2,3}} );
REQUIRE( foo1b_vals == std::list<int>{} );
REQUIRE( foo2c_vals == std::list{{-5, 4}} );
REQUIRE( var::get<bt_list>(a.at("bar")).empty() );
}
TEST_CASE("bt tuple serialization", "[bt][tuple][serialization]") {
// Deserializing directly into a tuple:
std::tuple<int, std::string, std::vector<int>> x{42, "hi", {{1,2,3,4,5}}};
REQUIRE( bt_serialize(x) == "li42e2:hili1ei2ei3ei4ei5eee" );
using Y = std::tuple<std::string, std::string, std::unordered_map<std::string, int>>;
REQUIRE( bt_deserialize<Y>("l5:hello3:omgd1:ai1e1:bi2eee")
== Y{"hello", "omg", {{"a",1}, {"b",2}}} );
using Z = std::tuple<std::tuple<int, std::string, std::string>, std::pair<int, int>>;
Z z{{3, "abc", "def"}, {4, 5}};
REQUIRE( bt_serialize(z) == "lli3e3:abc3:defeli4ei5eee" );
REQUIRE( bt_deserialize<Z>("lli6e3:ghi3:jkleli7ei8eee") == Z{{6, "ghi", "jkl"}, {7, 8}} );
using W = std::pair<std::string, std::pair<int, unsigned>>;
REQUIRE( bt_serialize(W{"zzzzzzzzzz", {42, 42}}) == "l10:zzzzzzzzzzli42ei42eee" );
REQUIRE_THROWS( bt_deserialize<std::tuple<int>>("li1e") ); // missing closing e
REQUIRE_THROWS( bt_deserialize<std::pair<int, int>>("li1ei-4e") ); // missing closing e
REQUIRE_THROWS( bt_deserialize<std::tuple<int>>("li1ei2ee") ); // too many elements
REQUIRE_THROWS( bt_deserialize<std::pair<int, int>>("li1ei-2e0:e") ); // too many elements
REQUIRE_THROWS( bt_deserialize<std::tuple<int, int>>("li1ee") ); // too few elements
REQUIRE_THROWS( bt_deserialize<std::pair<int, int>>("li1ee") ); // too few elements
REQUIRE_THROWS( bt_deserialize<std::tuple<std::string>>("li1ee") ); // wrong element type
REQUIRE_THROWS( bt_deserialize<std::pair<int, std::string>>("li1ei8ee") ); // wrong element type
REQUIRE_THROWS( bt_deserialize<std::pair<int, std::string>>("l1:x1:xe") ); // wrong element type
// Converting from a generic bt_value/bt_list:
bt_value a = bt_get("l5:hello3:omgi12345ee");
using V1 = std::tuple<std::string, std::string_view, uint16_t>;
REQUIRE( get_tuple<V1>(a) == V1{"hello", "omg"sv, 12345} );
bt_value b = bt_get("l5:hellod1:ai1e1:bi2eee");
using V2 = std::pair<std::string_view, bt_dict>;
REQUIRE( get_tuple<V2>(b) == V2{"hello", {{"a",1U}, {"b",2U}}} );
bt_value c = bt_get("l5:helloi-4ed1:ai-1e1:bi-2eee");
using V3 = std::tuple<std::string, int, bt_dict>;
REQUIRE( get_tuple<V3>(c) == V3{"hello", -4, {{"a",-1}, {"b",-2}}} );
REQUIRE_THROWS( get_tuple<V1>(bt_get("l5:hello3:omge")) ); // too few
REQUIRE_THROWS( get_tuple<V1>(bt_get("l5:hello3:omgi1ei1ee")) ); // too many
REQUIRE_THROWS( get_tuple<V1>(bt_get("l5:helloi1ei1ee")) ); // wrong type
// Construct a bt_value from tuples:
bt_value l{std::make_tuple(3, 4, "hi"sv)};
REQUIRE( bt_serialize(l) == "li3ei4e2:hie" );
bt_list m{{1, 2, std::make_tuple(3, 4, "hi"sv), std::make_pair("foo"s, "bar"sv), -4}};
REQUIRE( bt_serialize(m) == "li1ei2eli3ei4e2:hiel3:foo3:barei-4ee" );
}
TEST_CASE("bt allocation-free consumer", "[bt][dict][list][consumer]") {
// Consumer deserialization:
bt_list_consumer lc{"li1ei2eli3ei4e2:hiel3:foo3:barei-4ee"};
REQUIRE( lc.consume_integer<int>() == 1 );
REQUIRE( lc.consume_integer<int>() == 2 );
REQUIRE( lc.consume_list<std::tuple<int, int, std::string>>() == std::make_tuple(3, 4, "hi"s) );
REQUIRE( lc.consume_list<std::pair<std::string_view, std::string_view>>() == std::make_pair("foo"sv, "bar"sv) );
REQUIRE( lc.consume_integer<int>() == -4 );
bt_dict_consumer dc{"d1:Ai0e1:ali1e3:omge1:bli1ei2ei3eee"};
REQUIRE( dc.key() == "A" );
REQUIRE( dc.skip_until("a") );
REQUIRE( dc.next_list<std::pair<int8_t, std::string_view>>() ==
std::make_pair("a"sv, std::make_pair(int8_t{1}, "omg"sv)) );
REQUIRE( dc.next_list<std::tuple<int, int, int>>() ==
std::make_pair("b"sv, std::make_tuple(1, 2, 3)) );
}
TEST_CASE("bt allocation-free producer", "[bt][dict][list][producer]") {
char smallbuf[16];
bt_list_producer toosmall{smallbuf, 16}; // le, total = 2
toosmall += 42; // i42e, total = 6
toosmall += "abcdefgh"; // 8:abcdefgh, total=16
CHECK( toosmall.view() == "li42e8:abcdefghe" );
CHECK_THROWS_AS( toosmall += "", std::length_error );
char buf[1024];
bt_list_producer lp{buf, sizeof(buf)};
CHECK( lp.view() == "le" );
CHECK( (void*) lp.end() == (void*) (buf + 2) );
lp.append("abc");
CHECK( lp.view() == "l3:abce" );
lp += 42;
CHECK( lp.view() == "l3:abci42ee" );
std::vector<int> randos = {{1, 17, -999}};
lp.append(randos.begin(), randos.end());
CHECK( lp.view() == "l3:abci42ei1ei17ei-999ee" );
{
auto sublist = lp.append_list();
CHECK_THROWS_AS( lp.append(1), std::logic_error );
CHECK( sublist.view() == "le" );
CHECK( lp.view() == "l3:abci42ei1ei17ei-999elee" );
sublist.append(0);
auto sublist2{std::move(sublist)};
sublist2 += "";
CHECK( sublist2.view() == "li0e0:e" );
CHECK( lp.view() == "l3:abci42ei1ei17ei-999eli0e0:ee" );
}
lp.append_list().append_list().append_list() += "omg"s;
CHECK( lp.view() == "l3:abci42ei1ei17ei-999eli0e0:elll3:omgeeee" );
{
auto dict = lp.append_dict();
CHECK( dict.view() == "de" );
CHECK( lp.view() == "l3:abci42ei1ei17ei-999eli0e0:elll3:omgeeedee" );
CHECK_THROWS_AS( lp.append(1), std::logic_error );
dict.append("foo", "bar");
dict.append("g", 42);
CHECK( dict.view() == "d3:foo3:bar1:gi42ee" );
CHECK( lp.view() == "l3:abci42ei1ei17ei-999eli0e0:elll3:omgeeed3:foo3:bar1:gi42eee" );
dict.append_list("h").append_dict().append_dict("a").append_list("A") += 999;
CHECK( dict.view() == "d3:foo3:bar1:gi42e1:hld1:ad1:Ali999eeeeee" );
CHECK( lp.view() == "l3:abci42ei1ei17ei-999eli0e0:elll3:omgeeed3:foo3:bar1:gi42e1:hld1:ad1:Ali999eeeeeee" );
}
}
#ifdef OXENMQ_APPLE_TO_CHARS_WORKAROUND
TEST_CASE("apple to_chars workaround test", "[bt][apple][sucks]") {
char buf[20];
auto buf_view = [&](char* end) { return std::string_view{buf, static_cast<size_t>(end - buf)}; };
CHECK( buf_view(oxenmq::apple_to_chars10(buf, 0)) == "0" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, 1)) == "1" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, 2)) == "2" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, 10)) == "10" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, 42)) == "42" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, 99)) == "99" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, 1234567890)) == "1234567890" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, -1)) == "-1" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, -2)) == "-2" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, -10)) == "-10" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, -99)) == "-99" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, -1234567890)) == "-1234567890" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, char{42})) == "42" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, (unsigned char){42})) == "42" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, short{42})) == "42" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, std::numeric_limits<char>::min())) == "-128" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, std::numeric_limits<char>::max())) == "127" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, (unsigned char){42})) == "42" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, std::numeric_limits<uint64_t>::max())) == "18446744073709551615" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, int64_t{-1})) == "-1" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, std::numeric_limits<int64_t>::min())) == "-9223372036854775808" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, int64_t{-9223372036854775807})) == "-9223372036854775807" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, int64_t{9223372036854775807})) == "9223372036854775807" );
CHECK( buf_view(oxenmq::apple_to_chars10(buf, int64_t{9223372036854775806})) == "9223372036854775806" );
}
#endif

10
tests/test_commands.cpp
View File

@ -1,5 +1,5 @@
#include "common.h"
#include <oxenmq/hex.h>
#include <oxenc/hex.h>
#include <map>
#include <set>
@ -51,7 +51,7 @@ TEST_CASE("basic commands", "[commands]") {
REQUIRE( got );
REQUIRE( success );
REQUIRE_FALSE( failed );
REQUIRE( to_hex(pubkey) == to_hex(server.get_pubkey()) );
REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) );
}
client.send(c, "public.hello");
@ -61,7 +61,7 @@ TEST_CASE("basic commands", "[commands]") {
auto lock = catch_lock();
REQUIRE( hellos == 1 );
REQUIRE( his == 1 );
REQUIRE( to_hex(client_pubkey) == to_hex(client.get_pubkey()) );
REQUIRE( oxenc::to_hex(client_pubkey) == oxenc::to_hex(client.get_pubkey()) );
}
for (int i = 0; i < 50; i++)
@ -410,7 +410,7 @@ TEST_CASE("data parts", "[commands][send][data_parts]") {
REQUIRE( got );
REQUIRE( success );
REQUIRE_FALSE( failed );
REQUIRE( to_hex(pubkey) == to_hex(server.get_pubkey()) );
REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) );
}
std::vector some_data{{"abc"s, "def"s, "omg123\0zzz"s}};
@ -498,7 +498,7 @@ TEST_CASE("deferred replies", "[commands][send][deferred]") {
auto lock = catch_lock();
REQUIRE( connected );
REQUIRE_FALSE( failed );
REQUIRE( to_hex(pubkey) == to_hex(server.get_pubkey()) );
REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) );
}
std::unordered_set<std::string> replies;

1
tests/test_connect.cpp
View File

@ -1,5 +1,4 @@
#include "common.h"
#include <oxenmq/hex.h>
extern "C" {
#include <sodium.h>
}

427
tests/test_encoding.cpp
View File

@ -1,427 +0,0 @@
#include "oxenmq/hex.h"
#include "oxenmq/base32z.h"
#include "oxenmq/base64.h"
#include "common.h"
#include <iterator>
using namespace std::literals;
const std::string pk = "\xf1\x6b\xa5\x59\x10\x39\xf0\x89\xb4\x2a\x83\x41\x75\x09\x30\x94\x07\x4d\x0d\x93\x7a\x79\xe5\x3e\x5c\xe7\x30\xf9\x46\xe1\x4b\x88";
const std::string pk_hex = "f16ba5591039f089b42a834175093094074d0d937a79e53e5ce730f946e14b88";
const std::string pk_b32z = "6fi4kseo88aeupbkopyzknjo1odw4dcuxjh6kx1hhhax1tzbjqry";
const std::string pk_b64 = "8WulWRA58Im0KoNBdQkwlAdNDZN6eeU+XOcw+UbhS4g=";
TEST_CASE("hex encoding/decoding", "[encoding][decoding][hex]") {
REQUIRE( oxenmq::to_hex("\xff\x42\x12\x34") == "ff421234"s );
std::vector<uint8_t> chars{{1, 10, 100, 254}};
std::array<uint8_t, 8> out;
std::array<uint8_t, 8> expected{{'0', '1', '0', 'a', '6', '4', 'f', 'e'}};
oxenmq::to_hex(chars.begin(), chars.end(), out.begin());
REQUIRE( out == expected );
REQUIRE( oxenmq::to_hex(chars.begin(), chars.end()) == "010a64fe" );
REQUIRE( oxenmq::from_hex("12345678ffEDbca9") == "\x12\x34\x56\x78\xff\xed\xbc\xa9"s );
REQUIRE( oxenmq::is_hex("1234567890abcdefABCDEF1234567890abcdefABCDEF") );
REQUIRE_FALSE( oxenmq::is_hex("1234567890abcdefABCDEF1234567890aGcdefABCDEF") );
// ^
REQUIRE_FALSE( oxenmq::is_hex("1234567890abcdefABCDEF1234567890agcdefABCDEF") );
// ^
REQUIRE_FALSE( oxenmq::is_hex("\x11\xff") );
constexpr auto odd_hex = "1234567890abcdefABCDEF1234567890abcdefABCDE"sv;
REQUIRE_FALSE( oxenmq::is_hex(odd_hex) );
REQUIRE_FALSE( oxenmq::is_hex("0") );
REQUIRE( std::all_of(odd_hex.begin(), odd_hex.end(), oxenmq::is_hex_digit<char>) );
REQUIRE( oxenmq::from_hex(pk_hex) == pk );
REQUIRE( oxenmq::to_hex(pk) == pk_hex );
REQUIRE( oxenmq::from_hex(pk_hex.begin(), pk_hex.end()) == pk );
std::vector<std::byte> bytes{{std::byte{0xff}, std::byte{0x42}, std::byte{0x12}, std::byte{0x34}}};
std::basic_string_view<std::byte> b{bytes.data(), bytes.size()};
REQUIRE( oxenmq::to_hex(b) == "ff421234"s );
// In-place decoding and truncation via to_hex's returned iterator:
std::string some_hex = "48656c6c6f";
some_hex.erase(oxenmq::from_hex(some_hex.begin(), some_hex.end(), some_hex.begin()), some_hex.end());
REQUIRE( some_hex == "Hello" );
// Test the returned iterator from encoding
std::string hellohex;
*oxenmq::to_hex(some_hex.begin(), some_hex.end(), std::back_inserter(hellohex))++ = '!';
REQUIRE( hellohex == "48656c6c6f!" );
bytes.resize(8);
bytes[0] = std::byte{'f'}; bytes[1] = std::byte{'f'}; bytes[2] = std::byte{'4'}; bytes[3] = std::byte{'2'};
bytes[4] = std::byte{'1'}; bytes[5] = std::byte{'2'}; bytes[6] = std::byte{'3'}; bytes[7] = std::byte{'4'};
std::basic_string_view<std::byte> hex_bytes{bytes.data(), bytes.size()};
REQUIRE( oxenmq::is_hex(hex_bytes) );
REQUIRE( oxenmq::from_hex(hex_bytes) == "\xff\x42\x12\x34" );
REQUIRE( oxenmq::to_hex_size(1) == 2 );
REQUIRE( oxenmq::to_hex_size(2) == 4 );
REQUIRE( oxenmq::to_hex_size(3) == 6 );
REQUIRE( oxenmq::to_hex_size(4) == 8 );
REQUIRE( oxenmq::to_hex_size(100) == 200 );
REQUIRE( oxenmq::from_hex_size(2) == 1 );
REQUIRE( oxenmq::from_hex_size(4) == 2 );
REQUIRE( oxenmq::from_hex_size(6) == 3 );
REQUIRE( oxenmq::from_hex_size(98) == 49 );
}
TEST_CASE("base32z encoding/decoding", "[encoding][decoding][base32z]") {
REQUIRE( oxenmq::to_base32z("\0\0\0\0\0"s) == "yyyyyyyy" );
REQUIRE( oxenmq::to_base32z("\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef"sv)
== "yrtwk3hjixg66yjdeiuauk6p7hy1gtm8tgih55abrpnsxnpm3zzo");
REQUIRE( oxenmq::from_base32z("yrtwk3hjixg66yjdeiuauk6p7hy1gtm8tgih55abrpnsxnpm3zzo")
== "\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef"sv);
REQUIRE( oxenmq::from_base32z("YRTWK3HJIXG66YJDEIUAUK6P7HY1GTM8TGIH55ABRPNSXNPM3ZZO")
== "\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef\x01\x23\x45\x67\x89\xab\xcd\xef"sv);
auto five_nulls = oxenmq::from_base32z("yyyyyyyy");
REQUIRE( five_nulls.size() == 5 );
REQUIRE( five_nulls == "\0\0\0\0\0"s );
// 00000 00001 00010 00011 00100 00101 00110 00111
// ==
// 00000000 01000100 00110010 00010100 11000111
REQUIRE( oxenmq::from_base32z("ybndrfg8") == "\x00\x44\x32\x14\xc7"s );
// Special case 1: 7 base32z digits with 3 trailing 0 bits -> 4 bytes (the trailing 0s are dropped)
// 00000 00001 00010 00011 00100 00101 11000
// ==
// 00000000 01000100 00110010 00010111
REQUIRE( oxenmq::from_base32z("ybndrfa") == "\x00\x44\x32\x17"s );
// Round-trip it:
REQUIRE( oxenmq::from_base32z(oxenmq::to_base32z("\x00\x44\x32\x17"sv)) == "\x00\x44\x32\x17"sv );
REQUIRE( oxenmq::to_base32z(oxenmq::from_base32z("ybndrfa")) == "ybndrfa" );
// Special case 2: 7 base32z digits with 3 trailing bits 010; we just ignore the trailing stuff,
// as if it was specified as 0. (The last digit here is 11010 instead of 11000).
REQUIRE( oxenmq::from_base32z("ybndrf4") == "\x00\x44\x32\x17"s );
// This one won't round-trip to the same value since it has ignored garbage bytes at the end
REQUIRE( oxenmq::to_base32z(oxenmq::from_base32z("ybndrf4"s)) == "ybndrfa" );
REQUIRE( oxenmq::to_base32z(pk) == pk_b32z );
REQUIRE( oxenmq::to_base32z(pk.begin(), pk.end()) == pk_b32z );
REQUIRE( oxenmq::from_base32z(pk_b32z) == pk );
REQUIRE( oxenmq::from_base32z(pk_b32z.begin(), pk_b32z.end()) == pk );
std::string pk_b32z_again, pk_again;
oxenmq::to_base32z(pk.begin(), pk.end(), std::back_inserter(pk_b32z_again));
oxenmq::from_base32z(pk_b32z.begin(), pk_b32z.end(), std::back_inserter(pk_again));
REQUIRE( pk_b32z_again == pk_b32z );
REQUIRE( pk_again == pk );
// In-place decoding and truncation via returned iterator:
std::string some_b32z = "jb1sa5dx";
some_b32z.erase(oxenmq::from_base32z(some_b32z.begin(), some_b32z.end(), some_b32z.begin()), some_b32z.end());
REQUIRE( some_b32z == "Hello" );
// Test the returned iterator from encoding
std::string hellob32z;
*oxenmq::to_base32z(some_b32z.begin(), some_b32z.end(), std::back_inserter(hellob32z))++ = '!';
REQUIRE( hellob32z == "jb1sa5dx!" );
std::vector<std::byte> bytes{{std::byte{0}, std::byte{255}}};
std::basic_string_view<std::byte> b{bytes.data(), bytes.size()};
REQUIRE( oxenmq::to_base32z(b) == "yd9o" );
bytes.resize(4);
bytes[0] = std::byte{'y'}; bytes[1] = std::byte{'d'}; bytes[2] = std::byte{'9'}; bytes[3] = std::byte{'o'};
std::basic_string_view<std::byte> b32_bytes{bytes.data(), bytes.size()};
REQUIRE( oxenmq::is_base32z(b32_bytes) );
REQUIRE( oxenmq::from_base32z(b32_bytes) == "\x00\xff"sv );
REQUIRE( oxenmq::is_base32z("") );
REQUIRE_FALSE( oxenmq::is_base32z("y") );
REQUIRE( oxenmq::is_base32z("yy") );
REQUIRE_FALSE( oxenmq::is_base32z("yyy") );
REQUIRE( oxenmq::is_base32z("yyyy") );
REQUIRE( oxenmq::is_base32z("yyyyy") );
REQUIRE_FALSE( oxenmq::is_base32z("yyyyyy") );
REQUIRE( oxenmq::is_base32z("yyyyyyy") );
REQUIRE( oxenmq::is_base32z("yyyyyyyy") );
REQUIRE( oxenmq::to_base32z_size(1) == 2 );
REQUIRE( oxenmq::to_base32z_size(2) == 4 );
REQUIRE( oxenmq::to_base32z_size(3) == 5 );
REQUIRE( oxenmq::to_base32z_size(4) == 7 );
REQUIRE( oxenmq::to_base32z_size(5) == 8 );
REQUIRE( oxenmq::to_base32z_size(30) == 48 );
REQUIRE( oxenmq::to_base32z_size(31) == 50 );
REQUIRE( oxenmq::to_base32z_size(32) == 52 );
REQUIRE( oxenmq::to_base32z_size(33) == 53 );
REQUIRE( oxenmq::to_base32z_size(100) == 160 );
REQUIRE( oxenmq::from_base32z_size(160) == 100 );
REQUIRE( oxenmq::from_base32z_size(53) == 33 );
REQUIRE( oxenmq::from_base32z_size(52) == 32 );
REQUIRE( oxenmq::from_base32z_size(50) == 31 );
REQUIRE( oxenmq::from_base32z_size(48) == 30 );
REQUIRE( oxenmq::from_base32z_size(8) == 5 );
REQUIRE( oxenmq::from_base32z_size(7) == 4 );
REQUIRE( oxenmq::from_base32z_size(5) == 3 );
REQUIRE( oxenmq::from_base32z_size(4) == 2 );
REQUIRE( oxenmq::from_base32z_size(2) == 1 );
}
TEST_CASE("base64 encoding/decoding", "[encoding][decoding][base64]") {
// 00000000 00000000 00000000 -> 000000 000000 000000 000000
REQUIRE( oxenmq::to_base64("\0\0\0"s) == "AAAA" );
// 00000001 00000002 00000003 -> 000000 010000 000200 000003
REQUIRE( oxenmq::to_base64("\x01\x02\x03"s) == "AQID" );
REQUIRE( oxenmq::to_base64("\0\0\0\0"s) == "AAAAAA==" );
// 00000000 00000000 00000000 11111111 ->
// 000000 000000 000000 000000 111111 110000 (pad) (pad)
REQUIRE( oxenmq::to_base64("a") == "YQ==" );
REQUIRE( oxenmq::to_base64("ab") == "YWI=" );
REQUIRE( oxenmq::to_base64("abc") == "YWJj" );
REQUIRE( oxenmq::to_base64("abcd") == "YWJjZA==" );
REQUIRE( oxenmq::to_base64("abcde") == "YWJjZGU=" );
REQUIRE( oxenmq::to_base64("abcdef") == "YWJjZGVm" );
REQUIRE( oxenmq::to_base64_unpadded("a") == "YQ" );
REQUIRE( oxenmq::to_base64_unpadded("ab") == "YWI" );
REQUIRE( oxenmq::to_base64_unpadded("abc") == "YWJj" );
REQUIRE( oxenmq::to_base64_unpadded("abcd") == "YWJjZA" );
REQUIRE( oxenmq::to_base64_unpadded("abcde") == "YWJjZGU" );
REQUIRE( oxenmq::to_base64_unpadded("abcdef") == "YWJjZGVm" );
REQUIRE( oxenmq::to_base64("\0\0\0\xff"s) == "AAAA/w==" );
REQUIRE( oxenmq::to_base64("\0\0\0\xff\xff"s) == "AAAA//8=" );
REQUIRE( oxenmq::to_base64("\0\0\0\xff\xff\xff"s) == "AAAA////" );
REQUIRE( oxenmq::to_base64(
"Man is distinguished, not only by his reason, but by this singular passion from other "
"animals, which is a lust of the mind, that by a perseverance of delight in the "
"continued and indefatigable generation of knowledge, exceeds the short vehemence of "
"any carnal pleasure.")
==
"TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0aGlz"
"IHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1c3Qgb2Yg"
"dGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0aGUgY29udGlu"
"dWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdlLCBleGNlZWRzIHRo"
"ZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4=" );
REQUIRE( oxenmq::from_base64("A+/A") == "\x03\xef\xc0" );
REQUIRE( oxenmq::from_base64("YWJj") == "abc" );
REQUIRE( oxenmq::from_base64("YWJjZA==") == "abcd" );
REQUIRE( oxenmq::from_base64("YWJjZA") == "abcd" );
REQUIRE( oxenmq::from_base64("YWJjZB") == "abcd" ); // ignore superfluous bits
REQUIRE( oxenmq::from_base64("YWJjZB") == "abcd" ); // ignore superfluous bits
REQUIRE( oxenmq::from_base64("YWJj+") == "abc" ); // ignore superfluous bits
REQUIRE( oxenmq::from_base64("YWJjZGU=") == "abcde" );
REQUIRE( oxenmq::from_base64("YWJjZGU") == "abcde" );
REQUIRE( oxenmq::from_base64("YWJjZGVm") == "abcdef" );
REQUIRE( oxenmq::is_base64("YWJjZGVm") );
REQUIRE( oxenmq::is_base64("YWJjZGU") );
REQUIRE( oxenmq::is_base64("YWJjZGU=") );
REQUIRE( oxenmq::is_base64("YWJjZA==") );
REQUIRE( oxenmq::is_base64("YWJjZA") );
REQUIRE( oxenmq::is_base64("YWJjZB") ); // not really valid, but we explicitly accept it
REQUIRE_FALSE( oxenmq::is_base64("YWJjZ=") ); // invalid padding (padding can only be 4th or 3rd+4th of a 4-char block)
REQUIRE_FALSE( oxenmq::is_base64("YYYYA") ); // invalid: base64 can never be length 4n+1
REQUIRE_FALSE( oxenmq::is_base64("YWJj=") );
REQUIRE_FALSE( oxenmq::is_base64("YWJj=A") );
REQUIRE_FALSE( oxenmq::is_base64("YWJjA===") );
REQUIRE_FALSE( oxenmq::is_base64("YWJ[") );
REQUIRE_FALSE( oxenmq::is_base64("YWJ.") );
REQUIRE_FALSE( oxenmq::is_base64("_YWJ") );
REQUIRE( oxenmq::from_base64(
"TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0aGlz"
"IHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1c3Qgb2Yg"
"dGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0aGUgY29udGlu"
"dWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdlLCBleGNlZWRzIHRo"
"ZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4=" )
==
"Man is distinguished, not only by his reason, but by this singular passion from other "
"animals, which is a lust of the mind, that by a perseverance of delight in the "
"continued and indefatigable generation of knowledge, exceeds the short vehemence of "
"any carnal pleasure.");
REQUIRE( oxenmq::to_base64(pk) == pk_b64 );
REQUIRE( oxenmq::to_base64(pk.begin(), pk.end()) == pk_b64 );
REQUIRE( oxenmq::from_base64(pk_b64) == pk );
REQUIRE( oxenmq::from_base64(pk_b64.begin(), pk_b64.end()) == pk );
std::string pk_b64_again, pk_again;
oxenmq::to_base64(pk.begin(), pk.end(), std::back_inserter(pk_b64_again));
oxenmq::from_base64(pk_b64.begin(), pk_b64.end(), std::back_inserter(pk_again));
REQUIRE( pk_b64_again == pk_b64 );
REQUIRE( pk_again == pk );
// In-place decoding and truncation via returned iterator:
std::string some_b64 = "SGVsbG8=";
some_b64.erase(oxenmq::from_base64(some_b64.begin(), some_b64.end(), some_b64.begin()), some_b64.end());
REQUIRE( some_b64 == "Hello" );
// Test the returned iterator from encoding
std::string hellob64;
*oxenmq::to_base64(some_b64.begin(), some_b64.end(), std::back_inserter(hellob64))++ = '!';
REQUIRE( hellob64 == "SGVsbG8=!" );
std::vector<std::byte> bytes{{std::byte{0}, std::byte{255}}};
std::basic_string_view<std::byte> b{bytes.data(), bytes.size()};
REQUIRE( oxenmq::to_base64(b) == "AP8=" );
bytes.resize(4);
bytes[0] = std::byte{'/'}; bytes[1] = std::byte{'w'}; bytes[2] = std::byte{'A'}; bytes[3] = std::byte{'='};
std::basic_string_view<std::byte> b64_bytes{bytes.data(), bytes.size()};
REQUIRE( oxenmq::is_base64(b64_bytes) );
REQUIRE( oxenmq::from_base64(b64_bytes) == "\xff\x00"sv );
REQUIRE( oxenmq::to_base64_size(1) == 4 );
REQUIRE( oxenmq::to_base64_size(2) == 4 );
REQUIRE( oxenmq::to_base64_size(3) == 4 );
REQUIRE( oxenmq::to_base64_size(4) == 8 );
REQUIRE( oxenmq::to_base64_size(5) == 8 );
REQUIRE( oxenmq::to_base64_size(6) == 8 );
REQUIRE( oxenmq::to_base64_size(30) == 40 );
REQUIRE( oxenmq::to_base64_size(31) == 44 );
REQUIRE( oxenmq::to_base64_size(32) == 44 );
REQUIRE( oxenmq::to_base64_size(33) == 44 );
REQUIRE( oxenmq::to_base64_size(100) == 136 );
REQUIRE( oxenmq::from_base64_size(136) == 102 ); // Not symmetric because we don't know the last two are padding
REQUIRE( oxenmq::from_base64_size(134) == 100 ); // Unpadded
REQUIRE( oxenmq::from_base64_size(44) == 33 );
REQUIRE( oxenmq::from_base64_size(43) == 32 );
REQUIRE( oxenmq::from_base64_size(42) == 31 );
REQUIRE( oxenmq::from_base64_size(40) == 30 );
REQUIRE( oxenmq::from_base64_size(8) == 6 );
REQUIRE( oxenmq::from_base64_size(7) == 5 );
REQUIRE( oxenmq::from_base64_size(6) == 4 );
REQUIRE( oxenmq::from_base64_size(4) == 3 );
REQUIRE( oxenmq::from_base64_size(3) == 2 );
REQUIRE( oxenmq::from_base64_size(2) == 1 );
}
TEST_CASE("transcoding", "[decoding][encoding][base32z][hex][base64]") {
// Decoders:
oxenmq::base64_decoder in64{pk_b64.begin(), pk_b64.end()};
oxenmq::base32z_decoder in32z{pk_b32z.begin(), pk_b32z.end()};
oxenmq::hex_decoder in16{pk_hex.begin(), pk_hex.end()};
// Transcoders:
oxenmq::base32z_encoder b64_to_b32z{in64, in64.end()};
oxenmq::base32z_encoder hex_to_b32z{in16, in16.end()};
oxenmq::hex_encoder b64_to_hex{in64, in64.end()};
oxenmq::hex_encoder b32z_to_hex{in32z, in32z.end()};
oxenmq::base64_encoder hex_to_b64{in16, in16.end()};
oxenmq::base64_encoder b32z_to_b64{in32z, in32z.end()};
// These ones are stupid, but should work anyway:
oxenmq::base64_encoder b64_to_b64{in64, in64.end()};
oxenmq::base32z_encoder b32z_to_b32z{in32z, in32z.end()};
oxenmq::hex_encoder hex_to_hex{in16, in16.end()};
// Decoding to bytes:
std::string x;
auto xx = std::back_inserter(x);
std::copy(in64, in64.end(), xx);
REQUIRE( x == pk );
x.clear();
std::copy(in32z, in32z.end(), xx);
REQUIRE( x == pk );
x.clear();
std::copy(in16, in16.end(), xx);
REQUIRE( x == pk );
// Transcoding
x.clear();
std::copy(b64_to_hex, b64_to_hex.end(), xx);
CHECK( x == pk_hex );
x.clear();
std::copy(b64_to_b32z, b64_to_b32z.end(), xx);
CHECK( x == pk_b32z );
x.clear();
std::copy(b64_to_b64, b64_to_b64.end(), xx);
CHECK( x == pk_b64 );
x.clear();
std::copy(b32z_to_hex, b32z_to_hex.end(), xx);
CHECK( x == pk_hex );
x.clear();
std::copy(b32z_to_b32z, b32z_to_b32z.end(), xx);
CHECK( x == pk_b32z );
x.clear();
std::copy(b32z_to_b64, b32z_to_b64.end(), xx);
CHECK( x == pk_b64 );
x.clear();
std::copy(hex_to_hex, hex_to_hex.end(), xx);
CHECK( x == pk_hex );
x.clear();
std::copy(hex_to_b32z, hex_to_b32z.end(), xx);
CHECK( x == pk_b32z );
x.clear();
std::copy(hex_to_b64, hex_to_b64.end(), xx);
CHECK( x == pk_b64 );
// Make a big chain of conversions
oxenmq::base32z_encoder it1{in64, in64.end()};
oxenmq::base32z_decoder it2{it1, it1.end()};
oxenmq::base64_encoder it3{it2, it2.end()};
oxenmq::base64_decoder it4{it3, it3.end()};
oxenmq::hex_encoder it5{it4, it4.end()};
x.clear();
std::copy(it5, it5.end(), xx);
CHECK( x == pk_hex );
// No-padding b64 encoding:
oxenmq::base64_encoder b64_nopad{pk.begin(), pk.end(), false};
x.clear();
std::copy(b64_nopad, b64_nopad.end(), xx);
CHECK( x == pk_b64.substr(0, pk_b64.size()-1) );
}
TEST_CASE("std::byte decoding", "[decoding][hex][base32z][base64]") {
// Decoding to std::byte is a little trickier because you can't assign to a byte without an
// explicit cast, which means we have to properly detect that output is going to a std::byte
// output.
// hex
auto b_in = "ff42"s;
std::vector<std::byte> b_out;
oxenmq::from_hex(b_in.begin(), b_in.end(), std::back_inserter(b_out));
REQUIRE( b_out == std::vector{std::byte{0xff}, std::byte{0x42}} );
b_out.emplace_back();
oxenmq::from_hex(b_in.begin(), b_in.end(), b_out.begin() + 1);
REQUIRE( b_out == std::vector{std::byte{0xff}, std::byte{0xff}, std::byte{0x42}} );
oxenmq::from_hex(b_in.begin(), b_in.end(), b_out.data());
REQUIRE( b_out == std::vector{std::byte{0xff}, std::byte{0x42}, std::byte{0x42}} );
// base32z
b_in = "yojky"s;
b_out.clear();
oxenmq::from_base32z(b_in.begin(), b_in.end(), std::back_inserter(b_out));
REQUIRE( b_out == std::vector{std::byte{0x04}, std::byte{0x12}, std::byte{0xa0}} );
b_out.emplace_back();
oxenmq::from_base32z(b_in.begin(), b_in.end(), b_out.begin() + 1);
REQUIRE( b_out == std::vector{std::byte{0x04}, std::byte{0x04}, std::byte{0x12}, std::byte{0xa0}} );
oxenmq::from_base32z(b_in.begin(), b_in.end(), b_out.data());
REQUIRE( b_out == std::vector{std::byte{0x04}, std::byte{0x12}, std::byte{0xa0}, std::byte{0xa0}} );
// base64
b_in = "yojk"s;
b_out.clear();
oxenmq::from_base64(b_in.begin(), b_in.end(), std::back_inserter(b_out));
REQUIRE( b_out == std::vector{std::byte{0xca}, std::byte{0x88}, std::byte{0xe4}} );
b_out.emplace_back();
oxenmq::from_base64(b_in.begin(), b_in.end(), b_out.begin() + 1);
REQUIRE( b_out == std::vector{std::byte{0xca}, std::byte{0xca}, std::byte{0x88}, std::byte{0xe4}} );
oxenmq::from_base64(b_in.begin(), b_in.end(), b_out.data());
REQUIRE( b_out == std::vector{std::byte{0xca}, std::byte{0x88}, std::byte{0xe4}, std::byte{0xe4}} );
}

1
tests/test_failures.cpp
View File

@ -1,5 +1,4 @@
#include "common.h"
#include <oxenmq/hex.h>
#include <map>
#include <set>

8
tests/test_requests.cpp
View File

@ -1,5 +1,5 @@
#include "common.h"
#include <oxenmq/hex.h>
#include <oxenc/hex.h>
using namespace oxenmq;
@ -39,7 +39,7 @@ TEST_CASE("basic requests", "[requests]") {
auto lock = catch_lock();
REQUIRE( connected );
REQUIRE_FALSE( failed );
REQUIRE( to_hex(pubkey) == to_hex(server.get_pubkey()) );
REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) );
}
std::atomic<bool> got_reply{false};
@ -102,7 +102,7 @@ TEST_CASE("request from server to client", "[requests]") {
REQUIRE( connected.load() );
REQUIRE( !failed.load() );
REQUIRE( i <= 1 );
REQUIRE( to_hex(pubkey) == to_hex(server.get_pubkey()) );
REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) );
}
std::atomic<bool> got_reply{false};
@ -157,7 +157,7 @@ TEST_CASE("request timeouts", "[requests][timeout]") {
REQUIRE( connected );
REQUIRE_FALSE( failed );
REQUIRE( to_hex(pubkey) == to_hex(server.get_pubkey()) );
REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) );
std::atomic<bool> got_triggered{false};
bool success;