oxen-io
/
oxen-core
mirror of https://github.com/oxen-io/oxen-core.git
2
1
Fork 0
Code Issues Projects Releases Wiki Activity

Convert quorumnet to loki-mq 

This adds the loki-mq dependency and replaces SNNetwork with it (along
with some syntax updates for how loki-mq changed a bit from SNNetwork).

This also replaces common/hex.h and common/string_view.h with loki-mq's
faster (hex) and more complete and tested (string_view) implementations.
This commit is contained in:
Jason Rhinelander 2020-03-03 00:38:18 -04:00
parent 5b97ff6e9c
commit c98688fd84
24 changed files with 204 additions and 5208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -15,3 +15,6 @@
[submodule "external/randomx"]
path = external/randomx
url = https://github.com/loki-project/RandomX
[submodule "external/loki-mq"]
path = external/loki-mq
url = https://github.com/loki-project/loki-mq.git

19
CMakeLists.txt
View File

@ -183,8 +183,18 @@ if(NOT MANUAL_SUBMODULES)
if (upToDate)
message(STATUS "Submodule '${relative_path}' is up-to-date")
else()
message(FATAL_ERROR "Submodule '${relative_path}' is not up-to-date. Please update with\ngit submodule update --init --force ${relative_path}\nor run cmake with -DMANUAL_SUBMODULES=1")
message(FATAL_ERROR "Submodule '${relative_path}' is not up-to-date. Please update with\ngit submodule update --init --recursive\nor run cmake with -DMANUAL_SUBMODULES=1")
endif()
# Extra arguments check nested submodules
foreach(submod ${ARGN})
execute_process(COMMAND git rev-parse "HEAD" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${relative_path}/${submod} OUTPUT_VARIABLE localHead)
execute_process(COMMAND git rev-parse "HEAD:${submod}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${relative_path} OUTPUT_VARIABLE checkedHead)
string(COMPARE EQUAL "${localHead}" "${checkedHead}" upToDate)
if (NOT upToDate)
message(FATAL_ERROR "Nested submodule '${relative_path}/${submod}' is not up-to-date. Please update with\ngit submodule update --init --recursive\nor run cmake with -DMANUAL_SUBMODULES=1")
endif()
endforeach()
endfunction ()
message(STATUS "Checking submodules")
@ -193,6 +203,7 @@ if(NOT MANUAL_SUBMODULES)
check_submodule(external/rapidjson)
check_submodule(external/trezor-common)
check_submodule(external/randomx)
check_submodule(external/loki-mq mapbox-variant cppzmq)
endif()
endif()
@ -275,7 +286,7 @@ endif()
# elseif(CMAKE_SYSTEM_NAME MATCHES ".*BSDI.*")
# set(BSDI TRUE)
include_directories(external/rapidjson/include src contrib/epee/include external/cppzmq external)
include_directories(external/rapidjson/include src contrib/epee/include external)
if(APPLE)
include_directories(SYSTEM /usr/include/malloc)
@ -853,10 +864,6 @@ if (LOKI_DEBUG_SHORT_PROOFS)
add_definitions(-DUPTIME_PROOF_BASE_MINUTE=3) # 20x faster uptime proofs
endif()
if(NOT ZMQ_LIBRARIES) # may be already set (such as by contrib/depends toolchain)
find_package(PkgConfig REQUIRED)
pkg_check_modules(ZMQ REQUIRED libzmq)
endif()
add_library(sodium INTERFACE)
if(NOT SODIUM_LIBRARIES)

1
external/CMakeLists.txt vendored
View File

@ -89,3 +89,4 @@ endif()
add_subdirectory(db_drivers)
add_subdirectory(easylogging++)
add_subdirectory(randomx EXCLUDE_FROM_ALL)
add_subdirectory(loki-mq)

17
external/cppzmq/LICENSE vendored
View File

@ -1,17 +0,0 @@
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.

2011
external/cppzmq/zmq.hpp vendored
View File

File diff suppressed because it is too large Load Diff

457
external/cppzmq/zmq_addon.hpp vendored
View File

@ -1,457 +0,0 @@
/*
Copyright (c) 2016-2017 ZeroMQ community
Copyright (c) 2016 VOCA AS / Harald Nøkland
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.
*/
#ifndef __ZMQ_ADDON_HPP_INCLUDED__
#define __ZMQ_ADDON_HPP_INCLUDED__
#include <zmq.hpp>
#include <deque>
#include <iomanip>
#include <sstream>
#include <stdexcept>
#ifdef ZMQ_CPP11
#include <functional>
#include <unordered_map>
#endif
namespace zmq
{
#ifdef ZMQ_HAS_RVALUE_REFS
/*
This class handles multipart messaging. It is the C++ equivalent of zmsg.h,
which is part of CZMQ (the high-level C binding). Furthermore, it is a major
improvement compared to zmsg.hpp, which is part of the examples in the ØMQ
Guide. Unnecessary copying is avoided by using move semantics to efficiently
add/remove parts.
*/
class multipart_t
{
private:
std::deque<message_t> m_parts;
public:
typedef std::deque<message_t>::iterator iterator;
typedef std::deque<message_t>::const_iterator const_iterator;
typedef std::deque<message_t>::reverse_iterator reverse_iterator;
typedef std::deque<message_t>::const_reverse_iterator const_reverse_iterator;
// Default constructor
multipart_t() {}
// Construct from socket receive
multipart_t(socket_t &socket) { recv(socket); }
// Construct from memory block
multipart_t(const void *src, size_t size) { addmem(src, size); }
// Construct from string
multipart_t(const std::string &string) { addstr(string); }
// Construct from message part
multipart_t(message_t &&message) { add(std::move(message)); }
// Move constructor
multipart_t(multipart_t &&other) { m_parts = std::move(other.m_parts); }
// Move assignment operator
multipart_t &operator=(multipart_t &&other)
{
m_parts = std::move(other.m_parts);
return *this;
}
// Destructor
virtual ~multipart_t() { clear(); }
message_t &operator[](size_t n) { return m_parts[n]; }
const message_t &operator[](size_t n) const { return m_parts[n]; }
message_t &at(size_t n) { return m_parts.at(n); }
const message_t &at(size_t n) const { return m_parts.at(n); }
iterator begin() { return m_parts.begin(); }
const_iterator begin() const { return m_parts.begin(); }
const_iterator cbegin() const { return m_parts.cbegin(); }
reverse_iterator rbegin() { return m_parts.rbegin(); }
const_reverse_iterator rbegin() const { return m_parts.rbegin(); }
iterator end() { return m_parts.end(); }
const_iterator end() const { return m_parts.end(); }
const_iterator cend() const { return m_parts.cend(); }
reverse_iterator rend() { return m_parts.rend(); }
const_reverse_iterator rend() const { return m_parts.rend(); }
// Delete all parts
void clear() { m_parts.clear(); }
// Get number of parts
size_t size() const { return m_parts.size(); }
// Check if number of parts is zero
bool empty() const { return m_parts.empty(); }
// Receive multipart message from socket
bool recv(socket_t &socket, int flags = 0)
{
clear();
bool more = true;
while (more) {
message_t message;
#ifdef ZMQ_CPP11
if (!socket.recv(message, static_cast<recv_flags>(flags)))
return false;
#else
if (!socket.recv(&message, flags))
return false;
#endif
more = message.more();
add(std::move(message));
}
return true;
}
// Send multipart message to socket
bool send(socket_t &socket, int flags = 0)
{
flags &= ~(ZMQ_SNDMORE);
bool more = size() > 0;
while (more) {
message_t message = pop();
more = size() > 0;
#ifdef ZMQ_CPP11
if (!socket.send(message,
static_cast<send_flags>((more ? ZMQ_SNDMORE : 0) | flags)))
return false;
#else
if (!socket.send(message, (more ? ZMQ_SNDMORE : 0) | flags))
return false;
#endif
}
clear();
return true;
}
// Concatenate other multipart to front
void prepend(multipart_t &&other)
{
while (!other.empty())
push(other.remove());
}
// Concatenate other multipart to back
void append(multipart_t &&other)
{
while (!other.empty())
add(other.pop());
}
// Push memory block to front
void pushmem(const void *src, size_t size)
{
m_parts.push_front(message_t(src, size));
}
// Push memory block to back
void addmem(const void *src, size_t size)
{
m_parts.push_back(message_t(src, size));
}
// Push string to front
void pushstr(const std::string &string)
{
m_parts.push_front(message_t(string.data(), string.size()));
}
// Push string to back
void addstr(const std::string &string)
{
m_parts.push_back(message_t(string.data(), string.size()));
}
// Push type (fixed-size) to front
template<typename T> void pushtyp(const T &type)
{
static_assert(!std::is_same<T, std::string>::value,
"Use pushstr() instead of pushtyp<std::string>()");
m_parts.push_front(message_t(&type, sizeof(type)));
}
// Push type (fixed-size) to back
template<typename T> void addtyp(const T &type)
{
static_assert(!std::is_same<T, std::string>::value,
"Use addstr() instead of addtyp<std::string>()");
m_parts.push_back(message_t(&type, sizeof(type)));
}
// Push message part to front
void push(message_t &&message) { m_parts.push_front(std::move(message)); }
// Push message part to back
void add(message_t &&message) { m_parts.push_back(std::move(message)); }
// Pop string from front
std::string popstr()
{
std::string string(m_parts.front().data<char>(), m_parts.front().size());
m_parts.pop_front();
return string;
}
// Pop type (fixed-size) from front
template<typename T> T poptyp()
{
static_assert(!std::is_same<T, std::string>::value,
"Use popstr() instead of poptyp<std::string>()");
if (sizeof(T) != m_parts.front().size())
throw std::runtime_error(
"Invalid type, size does not match the message size");
T type = *m_parts.front().data<T>();
m_parts.pop_front();
return type;
}
// Pop message part from front
message_t pop()
{
message_t message = std::move(m_parts.front());
m_parts.pop_front();
return message;
}
// Pop message part from back
message_t remove()
{
message_t message = std::move(m_parts.back());
m_parts.pop_back();
return message;
}
// get message part from front
const message_t &front()
{
return m_parts.front();
}
// get message part from back
const message_t &back()
{
return m_parts.back();
}
// Get pointer to a specific message part
const message_t *peek(size_t index) const { return &m_parts[index]; }
// Get a string copy of a specific message part
std::string peekstr(size_t index) const
{
std::string string(m_parts[index].data<char>(), m_parts[index].size());
return string;
}
// Peek type (fixed-size) from front
template<typename T> T peektyp(size_t index) const
{
static_assert(!std::is_same<T, std::string>::value,
"Use peekstr() instead of peektyp<std::string>()");
if (sizeof(T) != m_parts[index].size())
throw std::runtime_error(
"Invalid type, size does not match the message size");
T type = *m_parts[index].data<T>();
return type;
}
// Create multipart from type (fixed-size)
template<typename T> static multipart_t create(const T &type)
{
multipart_t multipart;
multipart.addtyp(type);
return multipart;
}
// Copy multipart
multipart_t clone() const
{
multipart_t multipart;
for (size_t i = 0; i < size(); i++)
multipart.addmem(m_parts[i].data(), m_parts[i].size());
return multipart;
}
// Dump content to string
std::string str() const
{
std::stringstream ss;
for (size_t i = 0; i < m_parts.size(); i++) {
const unsigned char *data = m_parts[i].data<unsigned char>();
size_t size = m_parts[i].size();
// Dump the message as text or binary
bool isText = true;
for (size_t j = 0; j < size; j++) {
if (data[j] < 32 || data[j] > 127) {
isText = false;
break;
}
}
ss << "\n[" << std::dec << std::setw(3) << std::setfill('0') << size
<< "] ";
if (size >= 1000) {
ss << "... (to big to print)";
continue;
}
for (size_t j = 0; j < size; j++) {
if (isText)
ss << static_cast<char>(data[j]);
else
ss << std::hex << std::setw(2) << std::setfill('0')
<< static_cast<short>(data[j]);
}
}
return ss.str();
}
// Check if equal to other multipart
bool equal(const multipart_t *other) const
{
if (size() != other->size())
return false;
for (size_t i = 0; i < size(); i++)
if (*peek(i) != *other->peek(i))
return false;
return true;
}
private:
// Disable implicit copying (moving is more efficient)
multipart_t(const multipart_t &other) ZMQ_DELETED_FUNCTION;
void operator=(const multipart_t &other) ZMQ_DELETED_FUNCTION;
}; // class multipart_t
inline std::ostream &operator<<(std::ostream &os, const multipart_t &msg)
{
return os << msg.str();
}
#endif // ZMQ_HAS_RVALUE_REFS
#if defined(ZMQ_BUILD_DRAFT_API) && defined(ZMQ_CPP11) && defined(ZMQ_HAVE_POLLER)
class active_poller_t
{
public:
active_poller_t() = default;
~active_poller_t() = default;
active_poller_t(const active_poller_t &) = delete;
active_poller_t &operator=(const active_poller_t &) = delete;
active_poller_t(active_poller_t &&src) = default;
active_poller_t &operator=(active_poller_t &&src) = default;
using handler_type = std::function<void(event_flags)>;
void add(zmq::socket_ref socket, event_flags events, handler_type handler)
{
auto it = decltype(handlers)::iterator{};
auto inserted = bool{};
std::tie(it, inserted) =
handlers.emplace(socket,
std::make_shared<handler_type>(std::move(handler)));
try {
base_poller.add(socket, events,
inserted && *(it->second) ? it->second.get() : nullptr);
need_rebuild |= inserted;
}
catch (const zmq::error_t &) {
// rollback
if (inserted) {
handlers.erase(socket);
}
throw;
}
}
void remove(zmq::socket_ref socket)
{
base_poller.remove(socket);
handlers.erase(socket);
need_rebuild = true;
}
void modify(zmq::socket_ref socket, event_flags events)
{
base_poller.modify(socket, events);
}
size_t wait(std::chrono::milliseconds timeout)
{
if (need_rebuild) {
poller_events.resize(handlers.size());
poller_handlers.clear();
poller_handlers.reserve(handlers.size());
for (const auto &handler : handlers) {
poller_handlers.push_back(handler.second);
}
need_rebuild = false;
}
const auto count = base_poller.wait_all(poller_events, timeout);
std::for_each(poller_events.begin(), poller_events.begin() + static_cast<ptrdiff_t>(count),
[](decltype(base_poller)::event_type &event) {
if (event.user_data != nullptr)
(*event.user_data)(event.events);
});
return count;
}
ZMQ_NODISCARD bool empty() const noexcept { return handlers.empty(); }
size_t size() const noexcept { return handlers.size(); }
private:
bool need_rebuild{false};
poller_t<handler_type> base_poller{};
std::unordered_map<socket_ref, std::shared_ptr<handler_type>> handlers{};
std::vector<decltype(base_poller)::event_type> poller_events{};
std::vector<std::shared_ptr<handler_type>> poller_handlers{};
}; // class active_poller_t
#endif // defined(ZMQ_BUILD_DRAFT_API) && defined(ZMQ_CPP11) && defined(ZMQ_HAVE_POLLER)
} // namespace zmq
#endif // __ZMQ_ADDON_HPP_INCLUDED__

1
external/loki-mq vendored Submodule

@ -0,0 +1 @@
Subproject commit 512086613a79a014d5b18da28dff3828bf76a1aa

1
src/CMakeLists.txt
View File

@ -70,7 +70,6 @@ add_subdirectory(net)
add_subdirectory(mnemonics)
add_subdirectory(wallet)
add_subdirectory(cryptonote_protocol)
add_subdirectory(quorumnet)
if(NOT IOS)
if (NOT BUILD_INTEGRATION)

1
src/common/CMakeLists.txt
View File

@ -68,6 +68,7 @@ add_dependencies(common generate_translations_header)
target_link_libraries(common
PUBLIC
cncrypto
lokimq::lokimq
PRIVATE
libunbound
Boost::regex

36
src/common/hex.h
View File

@ -1,36 +0,0 @@
#pragma once
namespace hex
{
constexpr bool char_is_hex(char c)
{
bool result = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
return result;
}
constexpr char from_hex_digit(char x) {
return
(x >= '0' && x <= '9') ? x - '0' :
(x >= 'a' && x <= 'f') ? x - ('a' - 10):
(x >= 'A' && x <= 'F') ? x - ('A' - 10):
0;
}
constexpr char from_hex_pair(char a, char b) { return (from_hex_digit(a) << 4) | from_hex_digit(b); }
// Creates a string from a character sequence of hex digits. Undefined behaviour if any characters
// are not in [0-9a-fA-F] or if the input sequence length is not even.
template <typename It>
std::string from_hex(It begin, It end) {
using std::distance;
using std::next;
assert(distance(begin, end) % 2 == 0);
std::string raw;
raw.reserve(distance(begin, end) / 2);
while (begin != end) {
char a = *begin++;
char b = *begin++;
raw += from_hex_pair(a, b);
}
return raw;
}
}

40
src/common/string_view.h
View File

@ -1,40 +0,0 @@
#pragma once
#include <string>
#ifdef __cpp_lib_string_view
#include <string_view>
#endif
namespace common
{
#ifdef __cpp_lib_string_view
using string_view = std::string_view;
#else
class simple_string_view {
const char *_data;
size_t _size;
using char_traits = std::char_traits<char>;
public:
constexpr simple_string_view() noexcept : _data{nullptr}, _size{0} {}
constexpr simple_string_view(const simple_string_view &) noexcept = default;
simple_string_view(const std::string &str) : _data{str.data()}, _size{str.size()} {}
constexpr simple_string_view(const char *data, size_t size) noexcept : _data{data}, _size{size} {}
simple_string_view(const char *data) : _data{data}, _size{std::char_traits<char>::length(data)} {}
simple_string_view &operator=(const simple_string_view &) = default;
constexpr const char *data() const { return _data; }
constexpr size_t size() const { return _size; }
constexpr bool empty() { return _size == 0; }
operator std::string() const { return {_data, _size}; }
const char *begin() const { return _data; }
const char *end() const { return _data + _size; }
};
bool operator==(simple_string_view lhs, simple_string_view rhs) {
return lhs.size() == rhs.size() && 0 == std::char_traits<char>::compare(lhs.data(), rhs.data(), lhs.size());
};
bool operator!=(simple_string_view lhs, simple_string_view rhs) {
return !(lhs == rhs);
}
std::ostream &operator<<(std::ostream &os, const simple_string_view &s) { return os.write(s.data(), s.size()); }
using string_view = simple_string_view;
#endif
}; // namespace common

30
src/cryptonote_core/loki_name_system.cpp
View File

@ -2,7 +2,6 @@
#include "checkpoints/checkpoints.h"
#include "common/loki.h"
#include "common/hex.h"
#include "common/util.h"
#include "common/base32z.h"
#include "crypto/hash.h"
@ -12,6 +11,8 @@
#include "cryptonote_basic/tx_extra.h"
#include "cryptonote_core/blockchain.h"
#include <lokimq/hex.h>
#include <sqlite3.h>
extern "C"
@ -541,25 +542,20 @@ bool validate_lns_value(cryptonote::network_type nettype, mapping_type type, std
}
return false;
}
for (size_t val_index = 0; val_index < value.size(); val_index += 2)
if (!lokimq::is_hex(value))
{
char a = value[val_index];
char b = value[val_index + 1];
if (hex::char_is_hex(a) && hex::char_is_hex(b))
if (reason)
{
if (blob) // NOTE: Given blob, write the binary output
blob->buffer.data()[blob->len++] = hex::from_hex_pair(a, b);
}
else
{
if (reason)
{
err_stream << "LNS type=" << type <<", specifies name -> value mapping where the value is not a hex string given value=" << value;
*reason = err_stream.str();
}
return false;
err_stream << "LNS type=" << type <<", specifies name -> value mapping where the value is not a hex string given value=" << value;
*reason = err_stream.str();
}
return false;
}
if (blob) // NOTE: Given blob, write the binary output
{
blob->len = value.size() / 2;
assert(blob->len <= blob->buffer.size());
lokimq::from_hex(value.begin(), value.end(), blob->buffer.begin());
}
if (type == mapping_type::session)

1
src/cryptonote_protocol/CMakeLists.txt
View File

@ -38,7 +38,6 @@ target_link_libraries(cryptonote_protocol
PUBLIC
p2p
PRIVATE
quorumnet
miniupnpc
easylogging
extra)

279
src/cryptonote_protocol/quorumnet.cpp
View File

@ -1,4 +1,4 @@
// Copyright (c) 2019, The Loki Project
// Copyright (c) 2019-2020, The Loki Project
//
// All rights reserved.
//
@ -32,12 +32,13 @@
#include "cryptonote_core/service_node_rules.h"
#include "cryptonote_core/tx_blink.h"
#include "cryptonote_core/tx_pool.h"
#include "quorumnet/sn_network.h"
#include "quorumnet/conn_matrix.h"
#include "quorumnet_conn_matrix.h"
#include "cryptonote_config.h"
#include "common/random.h"
#include "common/lock.h"
#include <lokimq/lokimq.h>
#include <lokimq/hex.h>
#include <shared_mutex>
#undef LOKI_DEFAULT_LOG_CATEGORY
@ -48,8 +49,9 @@ namespace quorumnet {
namespace {
using namespace service_nodes;
using namespace std::string_literals;
using namespace std::chrono_literals;
using namespace std::literals;
using namespace lokimq;
using namespace lokimq::literals;
using blink_tx = cryptonote::blink_tx;
@ -67,7 +69,7 @@ struct pending_signature_hash {
using pending_signature_set = std::unordered_set<pending_signature, pending_signature_hash>;
struct SNNWrapper {
SNNetwork snn;
LokiMQ lmq;
cryptonote::core &core;
// Track submitted blink txes here; unlike the blinks stored in the mempool we store these ones
@ -89,7 +91,9 @@ struct SNNWrapper {
template <typename... Args>
SNNWrapper(cryptonote::core &core, Args &&...args) :
snn{std::forward<Args>(args)...}, core{core} {}
lmq{std::forward<Args>(args)...}, core{core} {
lmq.log_level(LogLevel::trace);
}
static SNNWrapper &from(void* obj) {
assert(obj);
@ -103,7 +107,7 @@ std::string get_data_as_string(const T &key) {
return {reinterpret_cast<const char *>(&key), sizeof(key)};
}
crypto::x25519_public_key x25519_from_string(const std::string &pubkey) {
crypto::x25519_public_key x25519_from_string(string_view pubkey) {
crypto::x25519_public_key x25519_pub = crypto::x25519_public_key::null();
if (pubkey.size() == sizeof(crypto::x25519_public_key))
std::memcpy(x25519_pub.data, pubkey.data(), pubkey.size());
@ -150,24 +154,24 @@ constexpr el::Level easylogging_level(LogLevel level) {
};
return el::Level::Unknown;
};
bool snn_want_log(LogLevel level) {
return ELPP->vRegistry()->allowed(easylogging_level(level), LOKI_DEFAULT_LOG_CATEGORY);
}
void snn_write_log(LogLevel level, const char *file, int line, std::string msg) {
el::base::Writer(easylogging_level(level), file, line, ELPP_FUNC, el::base::DispatchAction::NormalLog).construct(LOKI_DEFAULT_LOG_CATEGORY) << msg;
if (ELPP->vRegistry()->allowed(easylogging_level(level), LOKI_DEFAULT_LOG_CATEGORY))
el::base::Writer(easylogging_level(level), file, line, ELPP_FUNC, el::base::DispatchAction::NormalLog).construct(LOKI_DEFAULT_LOG_CATEGORY) << msg;
}
void setup_endpoints(SNNWrapper& snw);
void *new_snnwrapper(cryptonote::core &core, const std::string &bind) {
auto keys = core.get_service_node_keys();
auto peer_lookup = [&sn_list = core.get_service_node_list()](const std::string &x25519_pub) {
auto peer_lookup = [&sn_list = core.get_service_node_list()](string_view x25519_pub) {
return get_connect_string(sn_list, x25519_from_string(x25519_pub));
};
auto allow = [&sn_list = core.get_service_node_list()](const std::string &ip, const std::string &x25519_pubkey_str) {
auto allow = [&sn_list = core.get_service_node_list()](string_view ip, string_view x25519_pubkey_str) -> Allow {
auto x25519_pubkey = x25519_from_string(x25519_pubkey_str);
auto pubkey = sn_list.get_pubkey_from_x25519(x25519_pubkey);
MINFO("Accepting incoming " << (pubkey ? "SN" : "non-SN") << " connection authentication from ip/x25519/pubkey: " << ip << "/" << x25519_pubkey << "/" << pubkey);
if (pubkey) {
MINFO("Accepting incoming SN connection authentication from ip/x25519/pubkey: " << ip << "/" << x25519_pubkey << "/" << pubkey);
return SNNetwork::allow::service_node;
return {AuthLevel::none, true};
}
// Public connection:
@ -179,25 +183,29 @@ void *new_snnwrapper(cryptonote::core &core, const std::string &bind) {
// (In theory we could extend this to also only allow SN
// connections when in or near a blink/checkpoint/obligations/pulse quorum, but that
// would get messy fast and probably have little practical benefit).
return SNNetwork::allow::client;
return {AuthLevel::none, false};
};
SNNWrapper *obj;
if (!keys) {
MINFO("Starting remote-only quorumnet instance");
obj = new SNNWrapper(core, peer_lookup, allow, snn_want_log, snn_write_log);
} else {
std::string pubkey, seckey;
bool sn;
if (keys) {
MINFO("Starting quorumnet listener on " << bind << " with x25519 pubkey " << keys->pub_x25519);
obj = new SNNWrapper(core,
get_data_as_string(keys->pub_x25519),
get_data_as_string(keys->key_x25519.data),
std::vector<std::string>{{bind}},
peer_lookup,
allow,
snn_want_log, snn_write_log);
pubkey = get_data_as_string(keys->pub_x25519);
seckey = get_data_as_string(keys->key_x25519.data);
sn = true;
} else {
MINFO("Starting remote-only lokimq instance");
sn = false;
}
obj->snn.data = obj; // Provide pointer to the instance for callbacks
obj = new SNNWrapper(core, pubkey, seckey, sn, std::move(peer_lookup), snn_write_log);
setup_endpoints(*obj);
if (sn)
obj->lmq.listen_curve(bind, allow);
obj->lmq.start();
return obj;
}
@ -245,7 +253,7 @@ public:
/// Singleton wrapper around peer_info
peer_info(
SNNWrapper &snw,
SNNWrapper& snw,
quorum_type q_type,
std::shared_ptr<const quorum> &quorum,
bool opportunistic = true,
@ -264,13 +272,13 @@ public:
/// pubkey is always added to this exclude list.
template <typename QuorumIt>
peer_info(
SNNWrapper &snw,
SNNWrapper& snw,
quorum_type q_type,
QuorumIt qbegin, QuorumIt qend,
bool opportunistic = true,
std::unordered_set<crypto::public_key> exclude = {}
)
: snn{snw.snn} {
: lmq{snw.lmq} {
auto keys = snw.core.get_service_node_keys();
assert(keys);
@ -319,12 +327,12 @@ public:
/// Relays a command and any number of serialized data to everyone we're supposed to relay to
template <typename... T>
void relay_to_peers(const std::string &cmd, const T &...data) {
relay_to_peers_impl(cmd, std::array<send_option::serialized, sizeof...(T)>{send_option::serialized{data}...},
relay_to_peers_impl(cmd, std::array<std::string, sizeof...(T)>{bt_serialize(data)...},
std::make_index_sequence<sizeof...(T)>{});
}
private:
SNNetwork &snn;
LokiMQ &lmq;
/// Looks up a pubkey in known remotes and adds it to `peers`. If strong, it is added with an
/// address, otherwise it is added with an empty address. If the element already exists, it
@ -427,13 +435,13 @@ private:
/// Relays a command and pre-serialized data to everyone we're supposed to relay to
template<size_t N, size_t... I>
void relay_to_peers_impl(const std::string &cmd, std::array<send_option::serialized, N> relay_data, std::index_sequence<I...>) {
void relay_to_peers_impl(const std::string &cmd, std::array<std::string, N> relay_data, std::index_sequence<I...>) {
for (auto &peer : peers) {
MTRACE("Relaying " << cmd << " to peer " << as_hex(peer.first) << (peer.second.empty() ? " (if connected)"s : " @ " + peer.second));
MTRACE("Relaying " << cmd << " to peer " << to_hex(peer.first) << (peer.second.empty() ? " (if connected)"s : " @ " + peer.second));
if (peer.second.empty())
snn.send(peer.first, cmd, relay_data[I]..., send_option::optional{});
lmq.send(peer.first, cmd, relay_data[I]..., send_option::optional{});
else
snn.send(peer.first, cmd, relay_data[I]..., send_option::hint{peer.second});
lmq.send(peer.first, cmd, relay_data[I]..., send_option::hint{peer.second});
}
}
@ -458,8 +466,8 @@ bt_dict serialize_vote(const quorum_vote_t &vote) {
return result;
}
quorum_vote_t deserialize_vote(const bt_value &v) {
const auto &d = boost::get<bt_dict>(v); // throws if not a bt_dict
quorum_vote_t deserialize_vote(string_view v) {
const auto &d = bt_deserialize<bt_dict>(v); // throws if not a bt_dict
quorum_vote_t vote;
vote.version = get_int<uint8_t>(d.at("v"));
vote.type = get_enum<quorum_type>(d, "t");
@ -467,11 +475,11 @@ quorum_vote_t deserialize_vote(const bt_value &v) {
vote.group = get_enum<quorum_group>(d, "g");
if (vote.group == quorum_group::invalid) throw std::invalid_argument("invalid vote group");
vote.index_in_group = get_int<uint16_t>(d.at("i"));
auto &sig = boost::get<std::string>(d.at("s"));
auto &sig = d.at("s").get<std::string>();
if (sig.size() != sizeof(vote.signature)) throw std::invalid_argument("invalid vote signature size");
std::memcpy(&vote.signature, sig.data(), sizeof(vote.signature));
if (vote.type == quorum_type::checkpointing) {
auto &bh = boost::get<std::string>(d.at("bh"));
auto &bh = d.at("bh").get<std::string>();
if (bh.size() != sizeof(vote.checkpoint.block_hash.data)) throw std::invalid_argument("invalid vote checkpoint block hash");
std::memcpy(vote.checkpoint.block_hash.data, bh.data(), sizeof(vote.checkpoint.block_hash.data));
} else {
@ -525,10 +533,8 @@ void relay_obligation_votes(void *obj, const std::vector<service_nodes::quorum_v
snw.core.set_service_node_votes_relayed(relayed_votes);
}
void handle_obligation_vote(SNNetwork::message &m, void *self) {
auto &snw = SNNWrapper::from(self);
MDEBUG("Received a relayed obligation vote from " << as_hex(m.pubkey));
void handle_obligation_vote(Message& m, SNNWrapper& snw) {
MDEBUG("Received a relayed obligation vote from " << to_hex(m.conn.pubkey()));
if (m.data.size() != 1) {
MINFO("Ignoring vote: expected 1 data part, not " << m.data.size());
@ -558,10 +564,10 @@ void handle_obligation_vote(SNNetwork::message &m, void *self) {
}
if (vvc.m_added_to_pool)
relay_obligation_votes(self, std::move(vvote));
relay_obligation_votes(&snw, std::move(vvote));
}
catch (const std::exception &e) {
MWARNING("Deserialization of vote from " << as_hex(m.pubkey) << " failed: " << e.what());
MWARNING("Deserialization of vote from " << to_hex(m.conn.pubkey()) << " failed: " << e.what());
}
}
@ -791,10 +797,10 @@ void process_blink_signatures(SNNWrapper &snw, const std::shared_ptr<blink_tx> &
if (reply_tag && !reply_pubkey.empty()) {
if (became_approved) {
MINFO("Blink tx became approved; sending result back to originating node");
snw.snn.send(reply_pubkey, "bl_good", bt_dict{{"!", reply_tag}}, send_option::optional{});
snw.lmq.send(reply_pubkey, "bl_good", bt_serialize(bt_dict{{"!", reply_tag}}), send_option::optional{});
} else if (became_rejected) {
MINFO("Blink tx became rejected; sending result back to originating node");
snw.snn.send(reply_pubkey, "bl_bad", bt_dict{{"!", reply_tag}}, send_option::optional{});
snw.lmq.send(reply_pubkey, "bl_bad", bt_serialize(bt_dict{{"!", reply_tag}}), send_option::optional{});
}
}
}
@ -821,9 +827,7 @@ void process_blink_signatures(SNNWrapper &snw, const std::shared_ptr<blink_tx> &
/// "#" - precomputed tx hash. This much match the actual hash of the transaction (the blink
/// submission will fail immediately if it does not).
///
void handle_blink(SNNetwork::message &m, void *self) {
auto &snw = SNNWrapper::from(self);
void handle_blink(lokimq::Message& m, SNNWrapper& snw) {
// TODO: if someone sends an invalid tx (i.e. one that doesn't get to the distribution stage)
// then put a timeout on that IP during which new submissions from them are dropped for a short
// time.
@ -833,7 +837,7 @@ void handle_blink(SNNetwork::message &m, void *self) {
// message and close it.
// If an outgoing connection - refuse reconnections via ZAP and just close it.
MDEBUG("Received a blink tx from " << (m.sn ? "SN " : "non-SN ") << as_hex(m.pubkey));
MDEBUG("Received a blink tx from " << (m.conn.sn() ? "SN " : "non-SN ") << to_hex(m.conn.pubkey()));
auto keys = snw.core.get_service_node_keys();
assert(keys);
@ -844,7 +848,7 @@ void handle_blink(SNNetwork::message &m, void *self) {
// No valid data and so no reply tag; we can't send a response
return;
}
auto &data = boost::get<bt_dict>(m.data[0]);
auto data = bt_deserialize<bt_dict>(m.data[0]);
auto tag = get_or<uint64_t>(data, "!", 0);
@ -852,7 +856,7 @@ void handle_blink(SNNetwork::message &m, void *self) {
if (hf_version < HF_VERSION_BLINK) {
MWARNING("Rejecting blink message: blink is not available for hardfork " << (int) hf_version);
if (tag)
m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Invalid blink authorization height"}});
m.send_back("bl_nostart", bt_serialize(bt_dict{{"!", tag}, {"e", "Invalid blink authorization height"_sv}}));
return;
}
@ -863,14 +867,14 @@ void handle_blink(SNNetwork::message &m, void *self) {
if (blink_height < local_height - 2) {
MINFO("Rejecting blink tx because blink auth height is too low (" << blink_height << " vs. " << local_height << ")");
if (tag)
m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Invalid blink authorization height"}});
m.send_back("bl_nostart", bt_serialize(bt_dict{{"!", tag}, {"e", "Invalid blink authorization height"_sv}}));
return;
} else if (blink_height > local_height + 2) {
// TODO: if within some threshold (maybe 5-10?) we could hold it and process it once we are
// within 2.
MINFO("Rejecting blink tx because blink auth height is too high (" << blink_height << " vs. " << local_height << ")");
if (tag)
m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Invalid blink authorization height"}});
m.send_back("bl_nostart", bt_serialize(bt_dict{{"!", tag}, {"e", "Invalid blink authorization height"_sv}}));
return;
}
MTRACE("Blink tx auth height " << blink_height << " is valid (local height is " << local_height << ")");
@ -879,10 +883,10 @@ void handle_blink(SNNetwork::message &m, void *self) {
if (t_it == data.end()) {
MINFO("Rejecting blink tx: no tx data included in request");
if (tag)
m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "No transaction included in blink request"}});
m.send_back("bl_nostart", bt_serialize(bt_dict{{"!", tag}, {"e", "No transaction included in blink request"_sv}}));
return;
}
const std::string &tx_data = boost::get<std::string>(t_it->second);
const std::string &tx_data = t_it->second.get<std::string>();
MTRACE("Blink tx data is " << tx_data.size() << " bytes");
// "hash" is optional -- it lets us short-circuit processing the tx if we've already seen it,
@ -890,7 +894,7 @@ void handle_blink(SNNetwork::message &m, void *self) {
// the hash if we haven't seen it before -- this is only used to skip propagation and
// validation.
crypto::hash tx_hash;
auto &tx_hash_str = boost::get<std::string>(data.at("#"));
auto &tx_hash_str = data.at("#").get<std::string>();
bool already_approved = false, already_rejected = false;
if (tx_hash_str.size() == sizeof(crypto::hash)) {
std::memcpy(tx_hash.data, tx_hash_str.data(), sizeof(crypto::hash));
@ -915,7 +919,7 @@ void handle_blink(SNNetwork::message &m, void *self) {
// the reply until a signature comes in that flips it to approved/rejected
// status.
it->second.reply_tag = tag;
it->second.reply_pubkey = m.pubkey;
it->second.reply_pubkey = m.conn.pubkey();
return;
}
} else {
@ -924,16 +928,16 @@ void handle_blink(SNNetwork::message &m, void *self) {
}
}
}
MTRACE("Blink tx hash: " << as_hex(tx_hash.data));
MTRACE("Blink tx hash: " << to_hex(tx_hash.data));
} else {
MINFO("Rejecting blink tx: invalid tx hash included in request");
if (tag)
m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Invalid transaction hash"}});
m.send_back("bl_nostart", bt_serialize(bt_dict{{"!", tag}, {"e", "Invalid transaction hash"s}}));
return;
}
if (already_approved || already_rejected) {
snw.snn.send(m.pubkey, already_approved ? "bl_good" : "bl_bad", bt_dict{{"!", tag}}, send_option::optional{});
m.send_back(already_approved ? "bl_good" : "bl_bad", bt_serialize(bt_dict{{"!", tag}}), send_option::optional{});
return;
}
@ -944,12 +948,12 @@ void handle_blink(SNNetwork::message &m, void *self) {
} catch (const std::runtime_error &e) {
MINFO("Rejecting blink tx: " << e.what());
if (tag)
m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Unable to retrieve blink quorum: "s + e.what()}});
m.send_back("bl_nostart", bt_serialize(bt_dict{{"!", tag}, {"e", "Unable to retrieve blink quorum: "s + e.what()}}));
return;
}
peer_info pinfo{snw, quorum_type::blink, blink_quorums.begin(), blink_quorums.end(), true /*opportunistic*/,
{snw.core.get_service_node_list().get_pubkey_from_x25519(x25519_from_string(m.pubkey))} // exclude the peer that just sent it to us
{snw.core.get_service_node_list().get_pubkey_from_x25519(x25519_from_string(m.conn.pubkey()))} // exclude the peer that just sent it to us
};
if (pinfo.my_position_count > 0)
@ -957,7 +961,7 @@ void handle_blink(SNNetwork::message &m, void *self) {
else {
MINFO("Rejecting blink tx: this service node is not a member of the blink quorum!");
if (tag)
m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Blink tx relayed to non-blink quorum member"}});
m.send_back("bl_nostart", bt_serialize(bt_dict{{"!", tag}, {"e", "Blink tx relayed to non-blink quorum member"_sv}}));
return;
}
@ -974,7 +978,7 @@ void handle_blink(SNNetwork::message &m, void *self) {
if (!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash_actual)) {
MINFO("Rejecting blink tx: failed to parse transaction data");
if (tag)
m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Failed to parse transaction data"}});
m.send_back("bl_nostart", bt_serialize(bt_dict{{"!", tag}, {"e", "Failed to parse transaction data"_sv}}));
return;
}
MTRACE("Successfully parsed transaction data");
@ -982,7 +986,7 @@ void handle_blink(SNNetwork::message &m, void *self) {
if (tx_hash != tx_hash_actual) {
MINFO("Rejecting blink tx: submitted tx hash " << tx_hash << " did not match actual tx hash " << tx_hash_actual);
if (tag)
m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Invalid transaction hash"}});
m.send_back("bl_nostart", bt_serialize(bt_dict{{"!", tag}, {"e", "Invalid transaction hash"_sv}}));
return;
} else {
MTRACE("Pre-computed tx hash matches actual tx hash");
@ -994,7 +998,7 @@ void handle_blink(SNNetwork::message &m, void *self) {
if (!pinfo.strong_peers) {
MWARNING("Could not find connection info for any blink quorum peers. Aborting blink tx");
if (tag)
m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "No quorum peers are currently reachable"}});
m.send_back("bl_nostart", bt_serialize(bt_dict{{"!", tag}, {"e", "No quorum peers are currently reachable"_sv}}));
return;
}
@ -1014,7 +1018,7 @@ void handle_blink(SNNetwork::message &m, void *self) {
bl_info.pending_sigs.clear();
if (tag > 0) {
bl_info.reply_tag = tag;
bl_info.reply_pubkey = m.pubkey;
bl_info.reply_pubkey = m.conn.pubkey();
}
}
MTRACE("Accepted new blink tx for verification");
@ -1071,12 +1075,12 @@ void handle_blink(SNNetwork::message &m, void *self) {
signatures.emplace_back(approved, qi, pinfo.my_position[qi], sig);
}
process_blink_signatures(snw, btxptr, blink_quorums, checksum, std::move(signatures), tag, m.pubkey);
process_blink_signatures(snw, btxptr, blink_quorums, checksum, std::move(signatures), tag, m.conn.pubkey());
}
template <typename T, typename CopyValue>
void copy_signature_values(std::list<pending_signature> &signatures, const bt_value &val, CopyValue copy_value) {
auto &results = boost::get<bt_list>(val);
auto &results = val.get<bt_list>();
if (signatures.empty())
signatures.resize(results.size());
else if (results.empty())
@ -1110,15 +1114,13 @@ void copy_signature_values(std::list<pending_signature> &signatures, const bt_va
/// each list corresponds to the values at the same position of the other lists.
///
/// Signatures will be forwarded if new; known signatures will be ignored.
void handle_blink_signature(SNNetwork::message &m, void *self) {
auto &snw = SNNWrapper::from(self);
MDEBUG("Received a blink tx signature from SN " << as_hex(m.pubkey));
void handle_blink_signature(Message& m, SNNWrapper& snw) {
MDEBUG("Received a blink tx signature from SN " << to_hex(m.conn.pubkey()));
if (m.data.size() != 1)
throw std::runtime_error("Rejecting blink signature: expected one data entry not " + std::to_string(m.data.size()));
auto &data = boost::get<bt_dict>(m.data[0]);
auto data = bt_deserialize<bt_dict>(m.data[0]);
uint64_t blink_height = 0, checksum = 0;
crypto::hash tx_hash;
@ -1135,7 +1137,7 @@ void handle_blink_signature(SNNetwork::message &m, void *self) {
blink_height = get_int<uint64_t>(val);
break;
case '#': {
auto &hash_str = boost::get<std::string>(val);
auto &hash_str = val.get<std::string>();
if (hash_str.size() != sizeof(crypto::hash))
throw std::invalid_argument("Invalid blink signature data: invalid tx hash");
std::memcpy(tx_hash.data, hash_str.data(), sizeof(crypto::hash));
@ -1168,7 +1170,7 @@ void handle_blink_signature(SNNetwork::message &m, void *self) {
break;
case 's':
copy_signature_values<crypto::signature>(signatures, val, [](crypto::signature &dest, const bt_value &v) {
auto &sig_str = boost::get<std::string>(v);
auto& sig_str = v.get<std::string>();
if (sig_str.size() != sizeof(crypto::signature))
throw std::invalid_argument("Invalid blink signature data: invalid signature");
std::memcpy(&dest, sig_str.data(), sizeof(crypto::signature));
@ -1229,7 +1231,7 @@ void handle_blink_signature(SNNetwork::message &m, void *self) {
MINFO("Found blink tx in local blink cache");
process_blink_signatures(snw, btxptr, blink_quorums, checksum, std::move(signatures), reply_tag, reply_pubkey, m.pubkey);
process_blink_signatures(snw, btxptr, blink_quorums, checksum, std::move(signatures), reply_tag, reply_pubkey, m.conn.pubkey());
}
@ -1317,7 +1319,7 @@ std::future<std::pair<cryptonote::blink_result, std::string>> send_blink(void *o
return;
}
if (!proof.pubkey_x25519 || !proof.quorumnet_port || !proof.public_ip) {
MTRACE("Not including node " << pubkey << ": missing x25519(" << as_hex(get_data_as_string(proof.pubkey_x25519)) << "), "
MTRACE("Not including node " << pubkey << ": missing x25519(" << to_hex(get_data_as_string(proof.pubkey_x25519)) << "), "
"public_ip(" << epee::string_tools::get_ip_string_from_int32(proof.public_ip) << "), or qnet port(" << proof.quorumnet_port << ")");
return;
}
@ -1335,17 +1337,17 @@ std::future<std::pair<cryptonote::blink_result, std::string>> send_blink(void *o
indices.resize(4);
brd->remote_count = indices.size();
send_option::serialized data{bt_dict{
std::string data = bt_serialize<bt_dict>({
{"!", blink_tag},
{"#", get_data_as_string(tx_hash)},
{"h", height},
{"q", checksum},
{"t", tx_blob}
}};
});
for (size_t i : indices) {
MINFO("Relaying blink tx to " << as_hex(remotes[i].first) << " @ " << remotes[i].second);
snw.snn.send(remotes[i].first, "blink", data, send_option::hint{remotes[i].second});
MINFO("Relaying blink tx to " << to_hex(remotes[i].first) << " @ " << remotes[i].second);
snw.lmq.send(remotes[i].first, "blink", data, send_option::hint{remotes[i].second});
}
} catch (...) {
auto lock = tools::unique_lock(pending_blink_result_mutex);
@ -1404,14 +1406,14 @@ void common_blink_response(uint64_t tag, cryptonote::blink_result res, std::stri
///
/// It's possible for some nodes to accept and others to refuse, so we don't actually set the
/// promise unless we get a nostart response from a majority of the remotes.
void handle_blink_not_started(SNNetwork::message &m, void *) {
void handle_blink_not_started(Message& m) {
if (m.data.size() != 1) {
MERROR("Bad blink not started response: expected one data entry not " << m.data.size());
return;
}
auto &data = boost::get<bt_dict>(m.data[0]);
auto data = bt_deserialize<bt_dict>(m.data[0]);
auto tag = get_int<uint64_t>(data.at("!"));
auto &error = boost::get<std::string>(data.at("e"));
auto& error = data.at("e").get<std::string>();
MINFO("Received no-start blink response: " << error);
@ -1423,12 +1425,12 @@ void handle_blink_not_started(SNNetwork::message &m, void *) {
///
/// ! - the tag as included in the submission
///
void handle_blink_failure(SNNetwork::message &m, void *) {
void handle_blink_failure(Message &m) {
if (m.data.size() != 1) {
MERROR("Blink failure message not understood: expected one data entry not " << m.data.size());
return;
}
auto &data = boost::get<bt_dict>(m.data[0]);
auto data = bt_deserialize<bt_dict>(m.data[0]);
auto tag = get_int<uint64_t>(data.at("!"));
// TODO - we ought to be able to signal an error message *sometimes*, e.g. if one of the remotes
@ -1447,12 +1449,12 @@ void handle_blink_failure(SNNetwork::message &m, void *) {
///
/// ! - the tag as included in the submission
///
void handle_blink_success(SNNetwork::message &m, void *) {
void handle_blink_success(Message& m) {
if (m.data.size() != 1) {
MERROR("Blink success message not understood: expected one data entry not " << m.data.size());
return;
}
auto &data = boost::get<bt_dict>(m.data[0]);
auto data = bt_deserialize<bt_dict>(m.data[0]);
auto tag = get_int<uint64_t>(data.at("!"));
MINFO("Received blink success response");
@ -1460,19 +1462,19 @@ void handle_blink_success(SNNetwork::message &m, void *) {
common_blink_response(tag, cryptonote::blink_result::accepted, ""s);
}
void handle_ping(SNNetwork::message &m, void *) {
void handle_ping(Message& m) {
uint64_t tag = 0;
if (!m.data.empty()) {
auto &data = boost::get<bt_dict>(m.data[0]);
auto data = bt_deserialize<bt_dict>(m.data[0]);
tag = get_or<uint64_t>(data, "!", 0);
}
MINFO("Received ping request from " << (m.sn ? "SN" : "non-SN") << " " << as_hex(m.pubkey) << ", sending pong");
m.reply("pong", bt_dict{{"!", tag}, {"sn", m.sn}});
MINFO("Received ping request from " << (m.conn.sn() ? "SN" : "non-SN") << " " << to_hex(m.conn.pubkey()) << ", sending pong");
m.send_back("ping.pong", bt_serialize(bt_dict{{"!", tag}, {"sn", m.conn.sn()}}));
}
void handle_pong(SNNetwork::message &m, void *) {
MINFO("Received pong from " << (m.sn ? "SN" : "non-SN") << " " << as_hex(m.pubkey));
void handle_pong(Message& m) {
MINFO("Received pong from " << (m.conn.sn() ? "SN" : "non-SN") << " " << to_hex(m.conn.pubkey()));
}
} // end empty namespace
@ -1485,36 +1487,65 @@ void init_core_callbacks() {
cryptonote::quorumnet_delete = delete_snnwrapper;
cryptonote::quorumnet_relay_obligation_votes = relay_obligation_votes;
cryptonote::quorumnet_send_blink = send_blink;
}
// Receives an obligation vote
SNNetwork::register_command("vote_ob", SNNetwork::command_type::quorum, handle_obligation_vote);
namespace {
void setup_endpoints(SNNWrapper& snw) {
auto& lmq = snw.lmq;
// Receives a new blink tx submission from an external node, or forward from other quorum
// members who received it from an external node.
SNNetwork::register_command("blink", SNNetwork::command_type::public_, handle_blink);
// quorum.*: commands between quorum members, requires that both side of the connection is a SN
lmq.add_category("quorum", Access{AuthLevel::none, true /*remote sn*/, true /*local sn*/}, 2 /*reserved threads*/)
// Receives an obligation vote
.add_command("vote_ob", [&snw](Message& m) { handle_obligation_vote(m, snw); })
// Receives blink tx signatures or rejections between quorum members (either original or
// forwarded). These are propagated by the receiver if new
.add_command("blink_sign", [&snw](Message& m) { handle_blink_signature(m, snw); })
;
// Sends a message back to the blink initiator that the transaction was NOT relayed, either
// because the height was invalid or the quorum checksum failed. This is only sent by the entry
// point service nodes into the quorum to let it know the tx verification has not started from
// that node. It does not necessarily indicate a failure unless all entry point attempts return
// the same.
SNNetwork::register_command("bl_nostart", SNNetwork::command_type::response, handle_blink_not_started);
// blink.*: commands sent to blink quorum members from anyone (e.g. blink submission)
lmq.add_category("blink", Access{AuthLevel::none, false /*remote sn*/, true /*local sn*/}, 1 /*reserved thread*/)
// Receives a new blink tx submission from an external node, or forward from other quorum
// members who received it from an external node.
.add_command("submit", [&snw](Message& m) { handle_blink(m, snw); })
;
// Sends a message from the entry SNs back to the initiator that the Blink tx has been rejected:
// that is, enough signed rejections have occured that the Blink tx cannot be accepted.
SNNetwork::register_command("bl_bad", SNNetwork::command_type::response, handle_blink_failure);
// bl.*: responses to blinks sent from quorum members back to the node who submitted the blink
lmq.add_category("bl", Access{AuthLevel::none, true /*remote sn*/, false /*local sn*/})
// Message sent back to the blink initiator that the transaction was NOT relayed, either
// because the height was invalid or the quorum checksum failed. This is only sent by the
// entry point service nodes into the quorum to let it know the tx verification has not
// started from that node. It does not necessarily indicate a failure unless all entry
// point attempts return the same.
.add_command("nostart", handle_blink_not_started)
// Message send back from the entry SNs back to the initiator that the Blink tx has been
// rejected: that is, enough signed rejections have occured that the Blink tx cannot be
// accepted.
.add_command("bad", handle_blink_failure)
// Sends a message from the entry SNs back to the initiator that the Blink tx has been
// accepted and validated and is being broadcast to the network.
.add_command("good", handle_blink_success)
;
// Sends a message from the entry SNs back to the initiator that the Blink tx has been accepted
// and validated and is being broadcast to the network.
SNNetwork::register_command("bl_good", SNNetwork::command_type::response, handle_blink_success);
// ping.ping, ping.pong: triggers a reply with the auth status, used for quorumnet connectivity
// testing.
lmq.add_category("ping", Access{AuthLevel::none})
.add_command("ping", handle_ping)
.add_command("pong", handle_pong)
;
// Receives blink tx signatures or rejections between quorum members (either original or
// forwarded). These are propagated by the receiver if new
SNNetwork::register_command("blink_sign", SNNetwork::command_type::quorum, handle_blink_signature);
// Compatibility aliases. Transition plan:
// 7.x: keep the aliases and use them, since we need 6.x nodes to understand
// 8.x: keep the aliases (so the 7.x nodes still using them can talk to 8.x), but don't use them
// anymore.
// 8.x.1 (i.e. the first release after the 8.x hard fork): remove the aliases
// Replies with a pong and the auth status, used for quorumnet connectivity testing.
SNNetwork::register_command("ping", SNNetwork::command_type::public_, handle_ping);
SNNetwork::register_command("pong", SNNetwork::command_type::public_, handle_pong);
lmq.add_command_alias("vote_ob", "quorum.vote_ob");
lmq.add_command_alias("blink_sign", "quorum.blink_sign");
lmq.add_command_alias("blink", "blink.submit");
lmq.add_command_alias("bl_nostart", "bl.nostart");
lmq.add_command_alias("bl_bad", "bl.bad");
lmq.add_command_alias("bl_good", "bl.good");
}
}
}

6
src/cryptonote_protocol/quorumnet.h
View File

@ -1,4 +1,4 @@
// Copyright (c) 2019, The Loki Project
// Copyright (c) 2019-2020, The Loki Project
//
// All rights reserved.
//
@ -28,11 +28,9 @@
#pragma once
// This file (plus .cpp) contains the glue layer between cryptonote_core and quorumnet.
// This file (plus .cpp) contains the glue layer between cryptonote_core and loki-mq.
#include <vector>
//#include "quorumnet/service_node.h"
//
namespace service_nodes {
struct quorum_vote_t;

0
src/quorumnet/conn_matrix.h → src/cryptonote_protocol/quorumnet_conn_matrix.h
View File

33
src/quorumnet/CMakeLists.txt
View File

@ -1,33 +0,0 @@
# Copyright (c) 2019, The Loki Project
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are
# permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of
# conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list
# of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
add_library(quorumnet
sn_network.cpp
bt_serialize.cpp
)
target_link_libraries(quorumnet PRIVATE ${ZMQ_LIBRARIES})

145
src/quorumnet/bt_serialize.cpp
View File

@ -1,145 +0,0 @@
// Copyright (c) 2019, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "bt_serialize.h"
namespace quorumnet {
namespace detail {
template void bt_expect<bt_deserialize_invalid>(std::istream &is, char expect);
template void bt_expect<bt_deserialize_invalid_type>(std::istream &is, char expect);
void bt_deserialize<std::string>::operator()(std::istream &is, std::string &val) {
char first = is.peek();
bt_no_eof(is);
if (first < '0' || first > '9')
throw bt_deserialize_invalid_type("Expected 0-9 but found '" + std::string(1, first) + "' at input location " + std::to_string(is.tellg()));
uint64_t len;
is >> len;
char colon = is.peek();
if (colon == ':')
is.ignore();
else
throw bt_deserialize_invalid("Expected : but found '" + std::string(1, colon) + "' at input location " + std::to_string(is.tellg()));
if (len <= 4096) {
val.clear();
val.resize(len);
is.read(&val[0], len);
bt_no_eof(is);
return;
}
// Otherwise the serialization contains a large (>4K) value: don't trust the caller by
// pre-reserving (so that they can't run us out of memory by sending a malformed fake huge value
// to deserialize). Instead just let `val` reallocate as std::string sees fit.
val.clear();
char buffer[4096];
while (len) {
auto read = std::min<uint64_t>(len, 4096);
is.read(buffer, read);
bt_no_eof(is);
val.append(buffer, read);
len -= read;
}
}
// Check that we are on a 2's complement architecture. It's highly unlikely that this code ever
// runs on a non-2s-complement architecture, but check at compile time because we rely on these
// relations below.
static_assert(std::numeric_limits<int64_t>::min() + std::numeric_limits<int64_t>::max() == -1 &&
static_cast<uint64_t>(std::numeric_limits<int64_t>::max()) + uint64_t{1} == (uint64_t{1} << 63),
"Non 2s-complement architecture not supported!");
std::pair<maybe_signed_int64_t, bool> bt_deserialize_integer(std::istream &is) {
bt_expect<bt_deserialize_invalid_type>(is, 'i');
std::pair<maybe_signed_int64_t, bool> result;
char first = is.peek();
if (first == '-') {
result.second = true;
is.ignore();
first = is.peek();
}
bt_no_eof(is);
if (first < '0' || first > '9')
throw bt_deserialize_invalid("Expected 0-9 but found '" + std::string(1, first) + "' at input location " + std::to_string(is.tellg()));
uint64_t uval;
is >> uval;
bt_expect(is, 'e');
if (result.second) {
if (uval > (uint64_t{1} << 63))
throw bt_deserialize_invalid("Found too-large negative value just before input location " + std::to_string(is.tellg()));
result.first.i64 = -uval;
}
else {
result.first.u64 = uval;
}
return result;
}
template struct bt_deserialize<int64_t>;
template struct bt_deserialize<uint64_t>;
void bt_deserialize<bt_value, void>::operator()(std::istream &is, bt_value &val) {
auto next = is.peek();
bt_no_eof(is);
switch (next) {
case 'd': {
using dict_t = std::unordered_map<std::string, bt_value>;
dict_t dict;
bt_deserialize<dict_t>{}(is, dict);
val = std::move(dict);
break;
}
case 'l': {
using list_t = std::list<bt_value>;
list_t list;
bt_deserialize<list_t>{}(is, list);
val = std::move(list);
break;
}
case 'i': {
auto read = bt_deserialize_integer(is);
val = read.first.i64; // We only store an i64, but can get a u64 out of it via get<uint64_t>(val)
break;
}
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9': {
std::string str;
bt_deserialize<std::string>{}(is, str);
val = std::move(str);
break;
}
default:
throw bt_deserialize_invalid("Deserialize failed: encountered invalid value '" + std::string(1, next) + "'; expected one of [0-9idl]");
}
}
} // namespace detail
} // namespace quorumnet

596
src/quorumnet/bt_serialize.h
View File

@ -1,596 +0,0 @@
// Copyright (c) 2019, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
// Please no epee. *Especially* for serialization!
#include <string>
#ifdef __cpp_lib_string_view
#include <string_view>
#endif
#include <vector>
#include <list>
#include <unordered_map>
#include <algorithm>
#include <functional>
#include <ostream>
#include <istream>
#include <sstream>
#include <boost/variant.hpp>
#include "common.h"
#include "../common/osrb.h"
namespace quorumnet {
/** \file
* Quorumnet serialization is very simple: we support two primitive types, strings and integers,
* and two container types, lists and dicts with string keys. On the wire these go in BitTorrent
* byte encoding as described in BEP-0003 (https://www.bittorrent.org/beps/bep_0003.html#bencoding).
*
* On the C++ side, on input we allow strings, integral types, STL-like containers of these types,
* and STL-like containers of pairs with a string first value and any of these types as second
* value. We also accept boost::variants and std::variants (if compiled with std::variant support,
* i.e. in C++17 mode) that contain any of these.
*
* One minor deviation from BEP-0003 is that we don't support serializing values that don't fit in a
* 64-bit integer.
*
* On deserialization we can either deserialize into a boost::variant that supports everything, or
* we can fill a container of your given type (though this fails if the container isn't compatible
* with the deserialized data).
*/
/// Exception throw if deserialization fails
class bt_deserialize_invalid : public std::invalid_argument {
using std::invalid_argument::invalid_argument;
};
/// A more specific subclass that is thown if the serialization type is an initial mismatch: for
/// example, trying deserializing an int but the next thing in input is a list. This is not,
/// however, thrown if the type initially looks fine but, say, a nested serialization fails. This
/// error will only be thrown when the input stream has not been advanced (and so can be tried for a
/// different type).
class bt_deserialize_invalid_type : public bt_deserialize_invalid {
using bt_deserialize_invalid::bt_deserialize_invalid;
};
/// Recursive generic type that can store everything valid for a BT serialization.
using bt_value = boost::make_recursive_variant<
std::string, int64_t,
std::list<boost::recursive_variant_>,
std::unordered_map<std::string, boost::recursive_variant_>
>::type;
/// Convenience type that holds a dict of generic values (though *any* compatible data type can be used).
using bt_dict = std::unordered_map<std::string, bt_value>;
/// Convenience type that holds a list of generic values (though *any* compatible data type can be used).
using bt_list = std::list<bt_value>;
namespace detail {
// Fallback base case; we only get here if none of the partial specializations below work
template <typename T, typename SFINAE = void>
struct bt_serialize { static_assert(!std::is_same<T, T>::value, "Cannot serialize T: unsupported type for bt serialization"); };
template <typename T, typename SFINAE = void>
struct bt_deserialize { static_assert(!std::is_same<T, T>::value, "Cannot deserialize T: unsupported type for bt deserialization"); };
/// Checks that we aren't at EOF (or other !good() status) and throws if we are.
inline void bt_no_eof(std::istream &is) {
if (!is.good())
throw bt_deserialize_invalid(
std::string(is.eof() ? "Unexpected EOF" : "I/O error") + " while deserializing at location " +
std::to_string(is.tellg()));
}
/// Checks that the type code is the next character, and if so, consumes it. If not, throws.
template <typename Exception = bt_deserialize_invalid>
void bt_expect(std::istream &is, char expect) {
char t;
is.get(t);
bt_no_eof(is);
if (t != expect) {
is.unget();
throw Exception("Expected '" + std::string(1, expect) + "' but found '" + std::string(1, t) + "' at input location " + std::to_string(is.tellg()));
}
}
extern template void bt_expect<bt_deserialize_invalid>(std::istream &is, char expect);
extern template void bt_expect<bt_deserialize_invalid_type>(std::istream &is, char expect);
union maybe_signed_int64_t { int64_t i64; uint64_t u64; };
/// Deserializes a signed or unsigned 64. Sets the second bool to true iff the value is int64_t
/// because a negative value was read. Throws an exception if the read value doesn't fit in a
/// int64_t (if negative) or a uint64_t (if positive).
std::pair<maybe_signed_int64_t, bool> bt_deserialize_integer(std::istream &is);
/// Integer specializations
template <typename T>
struct bt_serialize<T, std::enable_if_t<std::is_integral<T>::value>> {
static_assert(sizeof(T) <= sizeof(uint64_t), "Serialization of integers larger than uint64_t is not supported");
void operator()(std::ostream &os, const T &val) {
// Cast 1-byte types to a larger type to avoid iostream interpreting them as single characters
using output_type = std::conditional_t<(sizeof(T) > 1), T, std::conditional_t<std::is_signed<T>::value, int, unsigned>>;
os << 'i' << static_cast<output_type>(val) << 'e';
}
};
template <typename T>
struct bt_deserialize<T, std::enable_if_t<std::is_integral<T>::value>> {
void operator()(std::istream &is, T &val) {
constexpr uint64_t umax = static_cast<uint64_t>(std::numeric_limits<T>::max());
constexpr int64_t smin = static_cast<int64_t>(std::numeric_limits<T>::min()),
smax = static_cast<int64_t>(std::numeric_limits<T>::max());
auto read = bt_deserialize_integer(is);
if (std::is_signed<T>::value) {
if (!read.second) { // read a positive value
if (read.first.u64 > umax)
throw bt_deserialize_invalid("Found too-large value " + std::to_string(read.first.u64) + " when deserializing just before input location " + std::to_string(is.tellg()));
val = static_cast<T>(read.first.u64);
} else {
bool oob = read.first.i64 < smin;
oob |= read.first.i64 > smax;
if (sizeof(T) < sizeof(int64_t) && oob)
throw bt_deserialize_invalid("Found out-of-range value " + std::to_string(read.first.i64) + " when deserializing just before input location " + std::to_string(is.tellg()));
val = static_cast<T>(read.first.i64);
}
} else {
if (read.second)
throw bt_deserialize_invalid("Found invalid negative value " + std::to_string(read.first.i64) + " when deserializing just before input location " + std::to_string(is.tellg()));
if (sizeof(T) < sizeof(uint64_t) && read.first.u64 > umax)
throw bt_deserialize_invalid("Found too-large value " + std::to_string(read.first.u64) + " when deserializing just before input location " + std::to_string(is.tellg()));
val = static_cast<T>(read.first.u64);
}
}
};
extern template struct bt_deserialize<int64_t>;
extern template struct bt_deserialize<uint64_t>;
/// String specialization
template <>
struct bt_serialize<std::string> {
void operator()(std::ostream &os, const std::string &val) { os << val.size() << ':' << val; }
};
template <>
struct bt_deserialize<std::string> {
void operator()(std::istream &is, std::string &val);
};
/// char * and string literals -- we allow serialization for convenience, but not deserialization
template <>
struct bt_serialize<char *> {
void operator()(std::ostream &os, const char *str) { auto len = std::strlen(str); os << len << ':'; os.write(str, len); }
};
template <size_t N>
struct bt_serialize<char[N]> {
void operator()(std::ostream &os, const char *str) { os << N-1 << ':'; os.write(str, N-1); }
};
/// Partial dict validity; we don't check the second type for serializability, that will be handled
/// via the base case static_assert if invalid.
template <typename T, typename = void> struct is_bt_input_dict_container : std::false_type {};
template <typename T>
struct is_bt_input_dict_container<T, std::enable_if_t<
std::is_same<std::string, std::remove_cv_t<typename T::value_type::first_type>>::value,
void_t<typename T::const_iterator /* is const iterable */,
typename T::value_type::second_type /* has a second type */>>>
: std::true_type {};
template <typename T, typename = void> struct is_bt_insertable : std::false_type {};
template <typename T>
struct is_bt_insertable<T,
void_t<decltype(std::declval<T>().insert(std::declval<T>().end(), std::declval<typename T::value_type>()))>>
: std::true_type {};
template <typename T, typename = void> struct is_bt_output_dict_container : std::false_type {};
template <typename T>
struct is_bt_output_dict_container<T, std::enable_if_t<
std::is_same<std::string, std::remove_cv_t<typename T::value_type::first_type>>::value &&
is_bt_insertable<T>::value,
void_t<typename T::value_type::second_type /* has a second type */>>>
: std::true_type {};
/// Specialization for a dict-like container (such as an unordered_map). We accept anything for a
/// dict that is const iterable over something that looks like a pair with std::string for first
/// value type. The value (i.e. second element of the pair) also must be serializable.
template <typename T>
struct bt_serialize<T, std::enable_if_t<is_bt_input_dict_container<T>::value>> {
using second_type = typename T::value_type::second_type;
using ref_pair = std::reference_wrapper<const typename T::value_type>;
void operator()(std::ostream &os, const T &dict) {
os << 'd';
std::vector<ref_pair> pairs;
pairs.reserve(dict.size());
for (const auto &pair : dict)
pairs.emplace(pairs.end(), pair);
std::sort(pairs.begin(), pairs.end(), [](ref_pair a, ref_pair b) { return a.get().first < b.get().first; });
for (auto &ref : pairs) {
bt_serialize<std::string>{}(os, ref.get().first);
bt_serialize<second_type>{}(os, ref.get().second);
}
os << 'e';
}
};
template <typename T>
struct bt_deserialize<T, std::enable_if_t<is_bt_output_dict_container<T>::value>> {
using second_type = typename T::value_type::second_type;
void operator()(std::istream &is, T &dict) {
bt_expect<bt_deserialize_invalid_type>(is, 'd');
dict.clear();
bt_deserialize<std::string> key_deserializer;
bt_deserialize<second_type> val_deserializer;
while (is.peek() != 'e') {
bt_no_eof(is);
std::string key;
second_type val;
try {
key_deserializer(is, key);
val_deserializer(is, val);
} catch (const bt_deserialize_invalid_type &e) {
// Rethrow a sub-element invalid type as a regular error (because the type *was* a list)
throw bt_deserialize_invalid(e.what());
}
dict.insert(dict.end(), typename T::value_type{std::move(key), std::move(val)});
}
bt_expect(is, 'e');
}
};
/// Accept anything that looks iterable; value serialization validity isn't checked here (it fails
/// via the base case static assert).
template <typename T, typename = void> struct is_bt_input_list_container : std::false_type {};
template <typename T>
struct is_bt_input_list_container<T, std::enable_if_t<
!std::is_same<T, std::string>::value &&
!is_bt_input_dict_container<T>::value,
void_t<typename T::const_iterator, typename T::value_type>>>
: std::true_type {};
template <typename T, typename = void> struct is_bt_output_list_container : std::false_type {};
template <typename T>
struct is_bt_output_list_container<T, std::enable_if_t<
!std::is_same<T, std::string>::value &&
!is_bt_output_dict_container<T>::value &&
is_bt_insertable<T>::value>>
: std::true_type {};
/// List specialization
template <typename T>
struct bt_serialize<T, std::enable_if_t<is_bt_input_list_container<T>::value>> {
void operator()(std::ostream &os, const T &list) {
os << 'l';
for (const auto &v : list)
bt_serialize<std::remove_cv_t<typename T::value_type>>{}(os, v);
os << 'e';
}
};
template <typename T>
struct bt_deserialize<T, std::enable_if_t<is_bt_output_list_container<T>::value>> {
using value_type = typename T::value_type;
void operator()(std::istream &is, T &list) {
bt_expect<bt_deserialize_invalid_type>(is, 'l');
list.clear();
bt_deserialize<value_type> deserializer;
while (is.peek() != 'e') {
bt_no_eof(is);
value_type v;
try {
deserializer(is, v);
} catch (const bt_deserialize_invalid_type &e) {
// Rethrow a sub-element invalid type as a regular error (because the type *was* a list)
throw bt_deserialize_invalid(e.what());
}
list.insert(list.end(), std::move(v));
}
bt_expect(is, 'e');
}
};
/// variant visitor; serializes whatever is contained
class bt_serialize_visitor {
std::ostream &os;
public:
using result_type = void;
bt_serialize_visitor(std::ostream &os) : os{os} {}
template <typename T> void operator()(const T &val) const {
bt_serialize<T>{}(os, val);
}
};
template <typename T>
using is_bt_deserializable = std::integral_constant<bool,
std::is_same<T, std::string>::value || std::is_integral<T>::value ||
is_bt_output_dict_container<T>::value || is_bt_output_list_container<T>::value>;
template <typename SFINAE, typename Variant, typename... Ts>
struct bt_deserialize_try_variant_impl {
void operator()(std::istream &, Variant &) {
throw bt_deserialize_invalid("Deserialization failed: could not deserialize value into any variant type");
}
};
template <typename... Ts, typename Variant>
void bt_deserialize_try_variant(std::istream &is, Variant &variant) {
bt_deserialize_try_variant_impl<void, Variant, Ts...>{}(is, variant);
}
template <typename Variant, typename T, typename... Ts>
struct bt_deserialize_try_variant_impl<std::enable_if_t<is_bt_deserializable<T>::value>, Variant, T, Ts...> {
void operator()(std::istream &is, Variant &variant) {
// Try to load the T variant. If deserialization fails with a invalid_type error then we
// can try to next one (that error leaves the istream in the same state).
try {
T val;
bt_deserialize<T>{}(is, val);
variant = std::move(val);
}
catch (bt_deserialize_invalid_type &e) {
bt_deserialize_try_variant<Ts...>(is, variant);
}
// Don't catch other exceptions: they aren't retriable failures
}
};
template <typename Variant, typename T, typename... Ts>
struct bt_deserialize_try_variant_impl<std::enable_if_t<!is_bt_deserializable<T>::value>, Variant, T, Ts...> {
void operator()(std::istream &is, Variant &variant) {
// Unsupported deserialization type, skip ahead
bt_deserialize_try_variant<Ts...>(is, variant);
}
};
/// Serialize a boost::variant
template <typename... Ts>
struct bt_serialize<boost::variant<Ts...>> {
void operator()(std::ostream &os, const boost::variant<Ts...> &val) {
boost::apply_visitor(bt_serialize_visitor{os}, val);
}
};
template <typename... Ts>
struct bt_deserialize<boost::variant<Ts...>> {
void operator()(std::istream &is, boost::variant<Ts...> &val) {
bt_deserialize_try_variant<Ts...>(is, val);
}
};
template <>
struct bt_deserialize<bt_value, void> {
void operator()(std::istream &is, bt_value &val);
};
#ifdef __cpp_lib_variant
/// C++17 std::variant support
template <typename... Ts>
struct bt_serialize<std::variant<Ts...>> {
void operator()(std::ostream &os, const std::variant<Ts...> &val) {
std::visit(bt_serialize_visitor{os}, val);
}
};
template <typename... Ts>
struct bt_deserialize<std::variant<Ts...>> {
void operator()(std::istream &is, std::variant<Ts...> &val) {
bt_deserialize_try_variant<Ts...>(is, val);
}
};
#endif
template <typename T>
struct bt_stream_serializer {
const T &val;
explicit bt_stream_serializer(const T &val) : val{val} {}
operator std::string() const {
std::ostringstream oss;
oss << *this;
return oss.str();
}
};
template <typename T>
std::ostream &operator<<(std::ostream &os, const bt_stream_serializer<T> &s) {
bt_serialize<T>{}(os, s.val);
return os;
}
template <typename T>
struct bt_stream_deserializer {
T &val;
bt_stream_deserializer(T &val) : val{val} {}
};
template <typename T>
std::istream &operator>>(std::istream &is, bt_stream_deserializer<T> &s) {
bt_deserialize<T>{}(is, s.val);
return is;
};
template <typename T>
std::istream &operator>>(std::istream &is, bt_stream_deserializer<T> &&s) {
bt_deserialize<T>{}(is, s.val);
return is;
};
} // namespace detail
/// Returns a wrapper around a value reference that can serialize the value directly to an output
/// stream or return it as a string. This class is intended to be used inline (i.e. without being
/// stored) as in:
///
/// int number = 42;
/// std::string encoded = bt_serialize(number);
/// // Equivalent:
/// //auto encoded = (std::string) bt_serialize(number);
///
/// std::list<int> my_list{{1,2,3}};
/// std::cout << bt_serialize(my_list);
///
/// While it is possible to store the returned object and use it, such as:
///
/// auto encoded = bt_serialize(42);
/// std::cout << encoded;
///
/// this approach is not generally recommended: the returned object stores a reference to the
/// passed-in type, which may not survive. If doing this note that it is the caller's
/// responsibility to ensure the serializer is not used past the end of the lifetime of the value
/// being serialized.
///
/// Also note that serializing directly to an output stream is more efficient as no intermediate
/// string containing the entire serialization has to be constructed.
///
template <typename T>
detail::bt_stream_serializer<T> bt_serializer(const T &val) { return detail::bt_stream_serializer<T>{val}; }
/// Serializes into a std::string. This is exactly equivalant to casting the above to a std::string.
template <typename T>
std::string bt_serialize(const T &val) { return bt_serializer(val); }
/// Returns a wrapper around a value non-const reference so that you can deserialize from an input
/// stream or a string using:
///
/// int value;
/// is >> bt_deserializer(value);
///
template <typename T>
detail::bt_stream_deserializer<T> bt_deserializer(T &val) { return detail::bt_stream_deserializer<T>{val}; }
/// Deserializes from a char * and size directly into `val`. Usage:
///
/// const char *encoded = "i42e";
/// size_t n = 4;
/// int value;
/// bt_deserialize(encoded, n, value); // Sets value to 42
template <typename T, std::enable_if_t<!std::is_const<T>::value, int> = 0>
void bt_deserialize(const char *data, size_t len, T &val) {
tools::one_shot_read_buffer buf{data, len};
std::istream is{&buf};
is >> bt_deserializer(val);
}
/// Deserializes the given string directly into `val`. Usage:
///
/// std::string encoded = "i42e";
/// int value;
/// bt_deserialize(encoded, value); // Sets value to 42
///
template <typename T, std::enable_if_t<!std::is_const<T>::value, int> = 0>
void bt_deserialize(
#ifdef __cpp_lib_string_view
std::string_view s,
#else
const std::string &s,
#endif
T &val) {
return bt_deserialize(s.data(), s.size(), val);
}
/// Deserializes the given string into a `T`, which is returned.
///
/// std::string encoded = "li1ei2ei3ee"; // bt-encoded list of ints: [1,2,3]
/// auto mylist = bt_deserialize<std::list<int>>(encoded);
///
template <typename T>
T bt_deserialize(
#ifdef __cpp_lib_string_view
std::string_view s
#else
const std::string &s
#endif
) {
T val;
bt_deserialize(s, val);
return val;
}
/// Deserializes the given C-style string into a `T`, which is returned.
///
/// char *encoded = "li1ei2ei3ee"; // bt-encoded list of ints: [1,2,3]
/// auto mylist = bt_deserialize_cstr<std::list<int>>(encoded, strlen(encoded));
///
template <typename T>
T bt_deserialize_cstr(const char *data, size_t len) {
T val;
bt_deserialize(data, len, val);
return val;
}
/// Deserializes the given value into a generic `bt_value` boost::variant which is capable of
/// holding all possible BT-encoded values.
///
/// Example:
///
/// std::string encoded = "i42e";
/// auto val = bt_get(encoded);
/// int v = get_int<int>(val); // fails unless the encoded value was actually an integer that
/// // fits into an `int`
///
inline bt_value bt_get(
#ifdef __cpp_lib_string_view
std::string_view s
#else
const std::string &s
#endif
) {
return bt_deserialize<bt_value>(s);
}
/// Helper functions to extract a value of some integral type from a bt_value which contains an
/// integer. Does range checking, throwing std::overflow_error if the stored value is outside the
/// range of the target type.
///
/// Example:
///
/// std::string encoded = "i123456789e";
/// auto val = bt_get(encoded);
/// auto v = get_int<uint32_t>(val); // throws if the decoded value doesn't fit in a uint32_t
template <typename IntType, std::enable_if_t<std::is_integral<IntType>::value, int> = 0>
IntType get_int(const bt_value &v) {
// It's highly unlikely that this code ever runs on a non-2s-complement architecture, but check
// at compile time if converting to a uint64_t (because while int64_t -> uint64_t is
// well-defined, uint64_t -> int64_t only does the right thing under 2's complement).
static_assert(!std::is_unsigned<IntType>::value || sizeof(IntType) != sizeof(int64_t) || -1 == ~0,
"Non 2s-complement architecture not supported!");
int64_t value = boost::get<int64_t>(v);
if (sizeof(IntType) < sizeof(int64_t)) {
if (value > static_cast<int64_t>(std::numeric_limits<IntType>::max())
|| value < static_cast<int64_t>(std::numeric_limits<IntType>::min()))
throw std::overflow_error("Unable to extract integer value: stored value is outside the range of the requested type");
}
return static_cast<IntType>(value);
}
}

44
src/quorumnet/common.h
View File

@ -1,44 +0,0 @@
// Copyright (c) 2019, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <type_traits>
namespace quorumnet {
// Various utility templates
#ifdef __cpp_lib_void_t
using std::void_t;
#else
/// C++17 void_t backport
template <typename... Ts> struct void_t_impl { using type = void; };
template <typename... Ts> using void_t = typename void_t_impl<Ts...>::type;
#endif
};

1061
src/quorumnet/sn_network.cpp
View File

File diff suppressed because it is too large Load Diff

592
src/quorumnet/sn_network.h
View File

@ -1,592 +0,0 @@
// Copyright (c) 2019, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
// Please no epee.
#include "zmq.hpp"
#include "bt_serialize.h"
#include <string>
#include <list>
#include <unordered_map>
#include <memory>
#include <functional>
#include <thread>
#include <mutex>
#include <iostream>
#include <chrono>
namespace quorumnet {
/// Logging levels passed into LogFunc
enum class LogLevel { trace, debug, info, warn, error, fatal };
using namespace std::chrono_literals;
/**
* Class that represents a listening service node on the quorum p2p network. This object supports
* connecting to service node peers and handles requests from node clients (for example, a remote
* node submitting a Blink transaction to a Blink quorum).
*
* Internally the class uses a worker thread to handle messages from other service nodes, which it
* then passed back into calling code via callbacks.
*/
class SNNetwork {
public:
/// The function type for looking up where to connect to the SN with the given pubkey. Should
/// return an empty string for an invalid or unknown pubkey or one without a known address.
using LookupFunc = std::function<std::string(const std::string &pubkey)>;
enum class allow { denied, client, service_node };
/// Callback type invoked to determine whether the given ip and pubkey are allowed to connect to
/// us as a SN, client, or not at all.
///
/// @param ip - the ip address of the incoming connection; will be empty if called to attempt to
/// "upgrade" the permission of an existing non-SN connection.
/// @param pubkey - the curve25519 pubkey (which is calculated from the SN ed25519 pubkey) of
/// the connecting service node (32 byte string), or an empty string if this is a client
/// connection without remote SN authentication.
/// @returns an `allow` enum value: `denied` if the connection is not allowed, `client` if the
/// connection is allowed as a client (i.e. not for SN-to-SN commands), `service_node` if the
/// connection is a valid SN.
using AllowFunc = std::function<allow(const std::string &ip, const std::string &pubkey)>;
/// Function pointer to ask whether a log of the given level is wanted. If it returns true the
/// log message will be built and then passed to Log.
using WantLog = bool (*)(LogLevel);
///
/// call to get somewhere to log to when there is a logging message. If it
/// returns a std::ostream pointer then output is sent to it; otherwise (i.e. nullptr) output is
/// suppressed. Takes three arguments: the log level, the __FILE__ value, and the __LINE__ value.
using WriteLog = void (*)(LogLevel level, const char *file, int line, std::string msg);
/// Explicitly non-copyable, non-movable because most things here aren't copyable, and a few
/// things aren't movable. If you need to pass the SNNetwork around, wrap it in a unique_ptr.
SNNetwork(const SNNetwork &) = delete;
SNNetwork &operator=(const SNNetwork &) = delete;
SNNetwork(SNNetwork &&) = delete;
SNNetwork &operator=(SNNetwork &&) = delete;
/// Encapsulates an incoming message from a remote node with message details plus extra info
/// need to send a reply back through the proxy thread via the `reply()` method.
class message {
private:
SNNetwork &net;
public:
std::string command; ///< The command name
std::vector<bt_value> data; ///< The provided command data parts, if any.
const std::string pubkey; ///< The originator pubkey (32 bytes)
const bool sn; ///< True if the pubkey is from a SN, meaning we can/should reconnect to it if necessary
/// Constructor
message(SNNetwork &net, std::string command, std::string pubkey, bool sn)
: net{net}, command{std::move(command)}, pubkey{std::move(pubkey)}, sn{sn} {
assert(this->pubkey.size() == 32);
}
/// Sends a reply. Arguments are forward to send() but with send_option::optional{} added
/// if the originator is not a SN. For SN messages (i.e. where `sn` is true) this is a
/// "strong" reply by default in that the proxy will establish a new connection to the SN if
/// no longer connected. For non-SN messages the reply will be attempted using the
/// available routing information, but if the connection has already been closed the reply
/// will be dropped.
template <typename... Args>
void reply(const std::string &command, Args &&...args);
};
/// Opaque pointer sent to the callbacks, to allow for including arbitrary state data (for
/// example, an owning object or state data). Defaults to nullptr and has to be explicitly set
/// if desired.
void *data = nullptr;
/// Possible command types; see register_command
enum class command_type { quorum, public_, response };
private:
zmq::context_t context;
/// A unique id for this SNNetwork, assigned in a thread-safe manner during construction.
const int object_id;
/// The keypair of this SN, either provided or generated during construction
std::string pubkey, privkey;
/// The thread in which most of the intermediate work happens (handling external connections
/// and proxying requests between them to worker threads)
std::thread proxy_thread;
/// Called to obtain a "command" socket that attaches to `control` to send commands to the
/// proxy thread from other threads. This socket is unique per thread and SNNetwork instance.
zmq::socket_t &get_control_socket();
/// Stores all of the sockets created in different threads via `get_control_socket`. This is
/// only used during destruction to close all of those open sockets, and is protected by an
/// internal mutex which is only locked by new threads getting a control socket and the
/// destructor.
std::vector<std::shared_ptr<zmq::socket_t>> thread_control_sockets;
///////////////////////////////////////////////////////////////////////////////////
/// NB: The following are all the domain of the proxy thread (once it is started)!
/// The lookup function that tells us where to connect to a peer
LookupFunc peer_lookup;
/// Our listening socket for public connections, or null for a remote-only
/// (TODO: change this to std::optional once we use C++17).
std::unique_ptr<zmq::socket_t> listener;
/// Callback to see whether the incoming connection is allowed
AllowFunc allow_connection;
/// Callback to see if we want a log message at the given level.
WantLog want_logs;
/// If want_logs returns true, the log message is build and passed into this.
WriteLog logger;
/// Info about a peer's established connection to us. Note that "established" means both
/// connected and authenticated.
struct peer_info {
/// True if we've authenticated this peer as a service node. (Note that new outgoing
/// connections are *always* expected to go to service nodes and will update this to true).
bool service_node = false;
/// Will be set to a non-empty routing prefix (which needs to be prefixed out outgoing
/// messages) if we have (or at least recently had) an established incoming connection with
/// this peer. Will be empty if there is no incoming connection.
std::string incoming;
/// The index in `remotes` if we have an established outgoing connection to this peer, -1 if
/// we have no outgoing connection to this peer.
int outgoing = -1;
/// The last time we sent or received a message (or had some other relevant activity) with
/// this peer. Used for closing outgoing connections that have reached an inactivity expiry
/// time.
std::chrono::steady_clock::time_point last_activity;
/// Updates last_activity to the current time
void activity() { last_activity = std::chrono::steady_clock::now(); }
/// After more than this much activity we will close an idle connection
std::chrono::milliseconds idle_expiry;
};
struct pk_hash { size_t operator()(const std::string &pubkey) const { size_t h; std::memcpy(&h, pubkey.data(), sizeof(h)); return h; } };
/// Currently peer connections, pubkey -> peer_info
std::unordered_map<std::string, peer_info, pk_hash> peers;
/// different polling sockets the proxy handler polls: this always contains some internal
/// sockets for inter-thread communication followed by listener socket and a pollitem for every
/// (outgoing) remote socket in `remotes`. This must be in a sequential vector because of zmq
/// requirements (otherwise it would be far nicer to not have to synchronize the two vectors).
std::vector<zmq::pollitem_t> pollitems;
/// Properly adds a socket to poll for input to pollitems
void add_pollitem(zmq::socket_t &sock);
/// The number of internal sockets in `pollitems`
static constexpr size_t poll_internal_size = 3;
/// The pollitems location corresponding to `remotes[0]`.
const size_t poll_remote_offset; // will be poll_internal_size + 1 for a full listener (the +1 is the listening socket); poll_internal_size for a remote-only
/// The outgoing remote connections we currently have open along with the remote pubkeys. Note
/// that the sockets here are generally accessed via the weak_ptr inside the `peers` element.
/// Each element [i] here corresponds to an the pollitem_t at pollitems[i+1+poll_internal_size].
/// (Ideally we'd use one structure, but zmq requires the pollitems be in contiguous storage).
std::vector<std::pair<std::string, zmq::socket_t>> remotes;
/// Socket we listen on to receive control messages in the proxy thread. Each thread has its own
/// internal "control" connection (returned by `get_control_socket()`) to this socket used to
/// give instructions to the proxy such as instructing it to initiate a connection to a remote
/// or send a message.
zmq::socket_t command{context, zmq::socket_type::router};
/// Router socket to reach internal worker threads from proxy
zmq::socket_t workers{context, zmq::socket_type::router};
/// Starts a new worker thread with the given id. Note that the worker may not yet be ready
/// until a READY signal arrives on the worker socket.
void spawn_worker(std::string worker_id);
/// Worker threads (ZMQ id -> thread)
std::unordered_map<std::string, std::thread> worker_threads;
/// ZMQ ids of idle, active workers
std::list<std::string> idle_workers;
/// Maximum number of worker threads created on demand up to this limit.
unsigned int max_workers;
/// Worker thread loop
void worker_thread(std::string worker_id);
/// Does the proxying work
void proxy_loop(const std::vector<std::string> &bind);
/// Forwards an incoming message to an idle worker, removing the idle worker from the queue
void proxy_to_worker(size_t conn_index, std::list<zmq::message_t> &&parts);
/// proxy thread command handlers for commands sent from the outer object QUIT. This doesn't
/// get called immediately on a QUIT command: the QUIT commands tells workers to quit, then this
/// gets called after all works have done so.
void proxy_quit();
/// Common connection implementation used by proxy_connect/proxy_send. Returns the socket
/// and, if a routing prefix is needed, the required prefix (or an empty string if not needed).
/// For an optional connect that fail, returns nullptr for the socket.
std::pair<zmq::socket_t *, std::string> proxy_connect(const std::string &pubkey, const std::string &connect_hint, bool optional, bool incoming_only, std::chrono::milliseconds keep_alive);
/// CONNECT command telling us to connect to a new pubkey. Returns the socket (which could be
/// existing or a new one).
std::pair<zmq::socket_t *, std::string> proxy_connect(bt_dict &&data);
/// DISCONNECT command telling us to disconnect out remote connection to the given pubkey (if we
/// have one).
void proxy_disconnect(const std::string &pubkey);
/// SEND command. Does a connect first, if necessary.
void proxy_send(bt_dict &&data);
/// REPLY command. Like SEND, but only has a listening socket route to send back to and so is
/// weaker (i.e. it cannot reconnect to the SN if the connection is no longer open).
void proxy_reply(bt_dict &&data);
/// ZAP (https://rfc.zeromq.org/spec:27/ZAP/) authentication handler; this is called with the
/// zap auth socket to do non-blocking processing of any waiting authentication requests waiting
/// on it to verify whether the connection is from a valid/allowed SN.
void process_zap_requests(zmq::socket_t &zap_auth);
/// Handles a control message from some outer thread to the proxy
void proxy_control_message(std::list<zmq::message_t> parts);
/// Closing any idle connections that have outlived their idle time. Note that this only
/// affects outgoing connections; incomings connections are the responsibility of the other end.
void proxy_expire_idle_peers();
/// Closes an outgoing connection immediately, updates internal variables appropriately.
/// Returns the next iterator (the original may or may not be removed from peers, depending on
/// whether or not it also has an active incoming connection).
decltype(peers)::iterator proxy_close_outgoing(decltype(peers)::iterator it);
/// End of proxy-specific members
///////////////////////////////////////////////////////////////////////////////////
/// Callbacks for data commands. Must be fully populated before starting SNNetwork instances
/// as this is accessed without a lock from worker threads.
///
/// The value is the {callback, public} pair where `public` is true if unauthenticated
/// connections may call this and false if only authenricated SNs may invoke the command.
static std::unordered_map<std::string, std::pair<void(*)(SNNetwork::message &message, void *data), command_type>> commands;
static bool commands_mutable;
/// Starts up the proxy thread; called during construction
void launch_proxy_thread(const std::vector<std::string> &bind);
public:
/**
* Constructs a SNNetwork connection listening on the given bind string.
*
* @param pubkey the service node's public key (32-byte binary string)
* @param privkey the service node's private key (32-byte binary string)
* @param bind list of addresses to bind to. Can be any string zmq supports; typically a tcp
* IP/port combination such as: "tcp://\*:4567" or "tcp://1.2.3.4:5678".
* @param peer_lookup function that takes a pubkey key (32-byte binary string) and returns a
* connection string such as "tcp://1.2.3.4:23456" to which a connection should be established
* to reach that service node. Note that this function is only called if there is no existing
* connection to that service node, and that the function is never called for a connection to
* self (that uses an internal connection instead).
* @param allow_incoming called on incoming connections with the (verified) incoming connection
* pubkey (32-byte binary string) to determine whether the given SN should be allowed to
* connect.
* @param want_log
* @param log a function pointer (or non-capturing lambda) to call to get a std::ostream pointer
* to send output to, or nullptr to suppress output. Optional; if omitted the default returns
* std::cerr for WARN and higher.
* @param max_workers the maximum number of simultaneous worker threads to start. Defaults to
* std::thread::hardware_concurrency(). Note that threads are only started on demand (i.e. when
* a request arrives when all existing threads are busy handling requests).
*/
SNNetwork(std::string pubkey,
std::string privkey,
const std::vector<std::string> &bind,
LookupFunc peer_lookup,
AllowFunc allow_connection,
WantLog want_log = [](LogLevel l) { return l >= LogLevel::warn; },
WriteLog logger = [](LogLevel, const char *f, int line, std::string msg) { std::cerr << f << ':' << line << ": " << msg << std::endl; },
unsigned int max_workers = std::thread::hardware_concurrency());
/** Constructs a SNNetwork that does not listen but can be used for connecting to remote
* listening service nodes, for example to submit blink transactions to service nodes. It
* generates a non-persistant x25519 key pair on startup (for encrypted communication with
* peers).
*/
SNNetwork(LookupFunc peer_lookup,
AllowFunc allow_connection,
WantLog want_log = [](LogLevel l) { return l >= LogLevel::warn; },
WriteLog logger = [](LogLevel, const char *f, int line, std::string msg) { std::cerr << f << ':' << line << ": " << msg << std::endl; },
unsigned int max_workers = std::thread::hardware_concurrency());
/**
* Destructor; instructs the proxy to quit. The proxy tells all workers to quit, waits for them
* to quit and rejoins the threads then quits itself. The outer thread (where the destructor is
* running) rejoins the proxy thread.
*/
~SNNetwork();
/**
* Returns true if we are running as a service node, which (currently) is synonymous with us
* being started in listening mode.
*/
bool is_service_node() const { return (bool) listener; }
/**
* Try to initiate a connection to the given SN in anticipation of needing a connection in the
* future. If a connection is already established, the connection's idle timer will be reset
* (so that the connection will not be closed too soon). If the given idle timeout is greater
* than the current idle timeout then the timeout increases to the new value; if less than the
* current timeout it is ignored. (Note that idle timeouts only apply if the existing
* connection is an outgoing connection).
*
* Note that this method (along with send) doesn't block waiting for a connection; it merely
* instructs the proxy thread that it should establish a connection.
*
* @param pubkey - the public key (32-byte binary string) of the service node to connect to
* @param keep_alive - the connection will be kept alive if there was valid activity within
* the past `keep_alive` milliseconds. If an outgoing connection already
* exists, the longer of the existing and the given keep alive is used.
* Note that the default applied here is much longer than the default for an
* implicit connect() by calling send() directly.
* @param hint - if non-empty and a new outgoing connection needs to be made this hint value
* may be used instead of calling the lookup function. (Note that there is no
* guarantee that the hint will be used; it should only be used when the lookup
* value has been precomputed to save a lookup call).
*/
void connect(const std::string &pubkey, std::chrono::milliseconds keep_alive = 5min, const std::string &hint = "");
/**
* Queue a message to be relayed to SN identified with the given pubkey without expecting a
* reply. The SN will attempt to relay (first connecting and handshaking if not already
* connected to the given SN).
*
* If a new connection it established it will have a relatively short (15s) idle timeout. If
* the connection should stay open longer you should call `connect(pubkey, IDLETIME)` first.
*
* Note that this method (along with connect) doesn't block waiting for a connection or for the
* message to send; it merely instructs the proxy thread that it should send.
*
* @param pubkey - the pubkey to send this to
* @param value - a bt_serializable value to serialize and send
* @param opts - any number of bt_serializable values and send options. Each send option affect
* how the send works; each serializable value becomes a serialized message part.
*
* Example:
*
* sn.send(pubkey, "hello", "abc", 42, send_option::hint("tcp://localhost:1234"));
*
* sends the command `hello` to the given pubkey, containing two additional message parts of
* serialized "abc" and 42 values, and, if not currently connected, using the given connection
* hint rather than performing a connection address lookup on the pubkey.
*/
template <typename... T>
void send(const std::string &pubkey, const std::string &cmd, const T &...opts);
/** The keep-alive time for a send() that results in a new connection. To use a longer
* keep-alive to a host call `connect()` first with the desired keep-alive time or pass the
* send_option::keep_alive
*/
static constexpr std::chrono::milliseconds default_send_keep_alive{15000};
/// The key pair this SN was created with
const std::string &get_pubkey() const { return pubkey; }
const std::string &get_privkey() const { return privkey; }
/**
* Registers a command that may be invoked on a quorumnet connection. The quorum command
* is one of three types:
* - `SNNetwork::command_type::quorum` - for a command that is only permitted between
* registered service nodes. It will be discarded if received from a remote that is not
* recognized as a service node, or if the local node is not a service node.
* - `SNNetwork::command_type::public_` - for a command that is permitted from anyone, but
* only if the local node is running as a service node.
* - `SNNetwork::command_type::response` - for a command that can be issued by a service
* node to the local client (whether or not running as a service node), typically issued
* in response to a `public_` command issued by this service node.
*
* Commands may only be registered before any SNNetwork instance has been constructed.
*
* @param command - the command string to assign. If it already exists it will be replaced.
* @param callback - a callback that takes the message info and the opaque `data` pointer
* @param cmd_type - the command type, as described above.
*/
static void register_command(
std::string command,
command_type cmd_type,
void(*callback)(SNNetwork::message &message, void *data));
};
/// Namespace for options to the send() method
namespace send_option {
/// `serialized` lets you serialize once when sending the same data to many peers by constructing a
/// single serialized option and passing it repeatedly rather than needing to reserialize on each
/// send.
struct serialized {
std::string data;
template <typename T>
serialized(const T &arg) : data{quorumnet::bt_serialize(arg)} {}
};
/// Specifies a connection hint when passed in to send(). If there is no current connection to the
/// peer then the hint is used to save a call to the LookupFunc to get the connection location.
/// (Note that there is no guarantee that the given hint will be used or that a LookupFunc call will
/// not be done.)
struct hint {
std::string connect_hint;
hint(std::string connect_hint) : connect_hint{std::move(connect_hint)} {}
};
/// Does a send() if we already have a connection (incoming or outgoing) with the given peer,
/// otherwise drops the message.
struct optional {};
/// Specifies that the message should be sent only if it can be sent on an existing incoming socket,
/// and dropped otherwise.
struct incoming {};
/// Specifies the idle timeout for the connection - if a new or existing outgoing connection is used
/// for the send and its current idle timeout setting is less than this value then it is updated.
struct keep_alive {
std::chrono::milliseconds time;
keep_alive(std::chrono::milliseconds time) : time{std::move(time)} {}
};
}
namespace detail {
// Sends a control message to the given socket consisting of the command plus optional dict
// data (only sent if the dict is non-empty).
void send_control(zmq::socket_t &sock, const std::string &cmd, const bt_dict &data = {});
/// Base case: takes a serializable value and appends it to the message parts
template <typename T>
void apply_send_option(bt_list &parts, bt_dict &, const T &arg) {
parts.push_back(quorumnet::bt_serialize(arg));
}
/// `serialized` specialization: lets you serialize once when sending the same data to many peers
template <> inline void apply_send_option(bt_list &parts, bt_dict &, const send_option::serialized &serialized) {
parts.push_back(serialized.data);
}
/// `hint` specialization: sets the hint in the control data
template <> inline void apply_send_option(bt_list &, bt_dict &control_data, const send_option::hint &hint) {
control_data["hint"] = hint.connect_hint;
}
/// `optional` specialization: sets the optional flag in the control data
template <> inline void apply_send_option(bt_list &, bt_dict &control_data, const send_option::optional &) {
control_data["optional"] = 1;
}
/// `incoming` specialization: sets the optional flag in the control data
template <> inline void apply_send_option(bt_list &, bt_dict &control_data, const send_option::incoming &) {
control_data["incoming"] = 1;
}
/// `keep_alive` specialization: increases the outgoing socket idle timeout (if shorter)
template <> inline void apply_send_option(bt_list &, bt_dict &control_data, const send_option::keep_alive &timeout) {
control_data["keep-alive"] = timeout.time.count();
}
/// Calls apply_send_option on each argument and returns a bt_dict with the command plus data stored
/// in the "send" key plus whatever else is implied by any given option arguments.
template <typename... T>
bt_dict send_control_data(const std::string &cmd, const T &...opts) {
bt_dict control_data;
bt_list parts{{cmd}};
#ifdef __cpp_fold_expressions
(detail::apply_send_option(parts, control_data, opts),...);
#else
(void) std::initializer_list<int>{(detail::apply_send_option(parts, control_data, opts), 0)...};
#endif
control_data["send"] = std::move(parts);
return control_data;
}
}
template <typename... T>
void SNNetwork::send(const std::string &pubkey, const std::string &cmd, const T &...opts) {
bt_dict control_data = detail::send_control_data(cmd, opts...);
control_data["pubkey"] = pubkey;
detail::send_control(get_control_socket(), "SEND", control_data);
}
template <typename... Args>
void SNNetwork::message::reply(const std::string &command, Args &&...args) {
if (sn) net.send(pubkey, command, std::forward<Args>(args)...);
else net.send(pubkey, command, send_option::optional{}, std::forward<Args>(args)...);
}
// Creates a hex string from a character sequence.
template <typename It>
std::string as_hex(It begin, It end) {
constexpr std::array<char, 16> lut{{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}};
std::string hex;
using std::distance;
hex.reserve(distance(begin, end) * 2);
while (begin != end) {
char c = *begin;
hex += lut[(c & 0xf0) >> 4];
hex += lut[c & 0x0f];
++begin;
}
return hex;
}
template <typename String>
inline std::string as_hex(const String &s) {
using std::begin;
using std::end;
return as_hex(begin(s), end(s));
}
}
// vim:sw=4:et

34
src/simplewallet/simplewallet.cpp
View File

@ -50,6 +50,8 @@
#include <boost/format.hpp>
#include <boost/regex.hpp>
#include <boost/range/adaptor/transformed.hpp>
#include <lokimq/hex.h>
#include <lokimq/string_view.h>
#include "include_base_utils.h"
#include "common/i18n.h"
#include "common/command_line.h"
@ -57,7 +59,6 @@
#include "common/dns_utils.h"
#include "common/base58.h"
#include "common/scoped_message_writer.h"
#include "common/hex.h"
#include "common/loki_integration_test_hooks.h"
#include "cryptonote_protocol/cryptonote_protocol_handler.h"
#include "cryptonote_core/service_node_voting.h"
@ -650,14 +651,9 @@ namespace
void print_secret_key(const crypto::secret_key &k)
{
static constexpr const char hex[] = u8"0123456789abcdef";
const uint8_t *ptr = (const uint8_t*)k.data;
for (size_t i = 0, sz = sizeof(k); i < sz; ++i)
{
putchar(hex[*ptr >> 4]);
putchar(hex[*ptr & 15]);
++ptr;
}
lokimq::string_view data{reinterpret_cast<const char*>(k.data), sizeof(k.data)};
std::ostream_iterator<char> osi{std::cout};
lokimq::to_hex(data.begin(), data.end(), osi);
}
}
@ -713,9 +709,9 @@ bool simple_wallet::viewkey(const std::vector<std::string> &args/* = std::vector
std::cout << "secret: On device. Not available" << std::endl;
} else {
SCOPED_WALLET_UNLOCK();
printf("secret: ");
std::cout << "secret: ";
print_secret_key(m_wallet->get_account().get_keys().m_view_secret_key);
putchar('\n');
std::cout << '\n';
}
std::cout << "public: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_account_address.m_view_public_key) << std::endl;
@ -735,9 +731,9 @@ bool simple_wallet::spendkey(const std::vector<std::string> &args/* = std::vecto
std::cout << "secret: On device. Not available" << std::endl;
} else {
SCOPED_WALLET_UNLOCK();
printf("secret: ");
std::cout << "secret: ";
print_secret_key(m_wallet->get_account().get_keys().m_spend_secret_key);
putchar('\n');
std::cout << '\n';
}
std::cout << "public: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_account_address.m_spend_public_key) << std::endl;
@ -4208,7 +4204,7 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr
PAUSE_READLINE();
std::cout << tr("View key: ");
print_secret_key(m_wallet->get_account().get_keys().m_view_secret_key);
putchar('\n');
std::cout << '\n';
}
catch (const std::exception& e)
{
@ -6644,14 +6640,10 @@ bool simple_wallet::print_lns_owners_to_names(const std::vector<std::string>& ar
fail_msg_writer() << "arg is not a 64 character ed25519 public key, arg = " << arg;
return false;
}
for (char c : arg)
if (!lokimq::is_hex(arg))
{
if (!hex::char_is_hex(c))
{
fail_msg_writer() << "arg contains a non-hex character = " << c << ", arg = " << arg;
return false;
}
fail_msg_writer() << "arg contains non-hex characters: " << arg;
return false;
}
request.entries.push_back(arg);
}

4
utils/qnet-ping.py Normal file → Executable file
View File

@ -68,11 +68,11 @@ for s in states:
socket.connect("tcp://{}:{}".format(ip, port))
print("Ping {}:{} (for SN {})".format(ip, port, pk))
bt_tag = bytes("i{}e".format(tag), "utf-8")
socket.send_multipart((b"ping", b"d1:!" + bt_tag + b"e"))
socket.send_multipart((b"ping.ping", b"d1:!" + bt_tag + b"e"))
ponged = False
while socket.poll(timeout=5000):
m = socket.recv_multipart()
if len(m) == 2 and m[0] == b'pong':
if len(m) == 2 and m[0] == b'ping.pong':
ponged = True
if m[1] == b'd1:!' + bt_tag + b'2:sni1ee':
print("Received pong, we were recognized as a SN")