From dd7a4104b5d5cc7be166bcdc52810768cb13c57b Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Sun, 27 Oct 2019 19:47:19 -0300 Subject: [PATCH] Blink This is the bulk of the work for blink. There is two pieces yet to come which will follow shortly, which are: the p2p communication of blink transactions (which needs to be fully synchronized, not just shared, unlike regular mempool txes); and an implementation of fee burning. Blink approval, multi-quorum signing, cli wallet and node support for submission denial are all implemented here. This overhauls and fixes various parts of the SNNetwork interface to fix some issues (particularly around non-SN communication with SNs, which wasn't working). There are also a few sundry FIXME's and TODO's of other minor details that will follow shortly under cleanup/testing/etc. --- src/cryptonote_config.h | 7 + src/cryptonote_core/cryptonote_core.cpp | 34 +- src/cryptonote_core/cryptonote_core.h | 37 +- src/cryptonote_core/service_node_list.cpp | 2 +- .../service_node_quorum_cop.cpp | 21 +- src/cryptonote_core/service_node_rules.h | 26 +- src/cryptonote_core/tx_blink.cpp | 128 +- src/cryptonote_core/tx_blink.h | 145 +- src/cryptonote_core/tx_pool.cpp | 37 +- src/cryptonote_core/tx_pool.h | 35 +- src/cryptonote_protocol/quorumnet.cpp | 1302 +++++++++++++---- src/daemon/rpc_command_executor.cpp | 4 +- src/quorumnet/sn_network.cpp | 564 +++---- src/quorumnet/sn_network.h | 183 ++- src/rpc/core_rpc_server.cpp | 21 + src/rpc/core_rpc_server_commands_defs.h | 6 +- src/simplewallet/simplewallet.cpp | 118 +- src/simplewallet/simplewallet.h | 17 +- src/wallet/wallet2.cpp | 75 +- src/wallet/wallet2.h | 7 +- 20 files changed, 1925 insertions(+), 844 deletions(-) diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 5bdb5577f..041afdaa1 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -100,6 +100,12 @@ static_assert(STAKING_PORTIONS % 3 == 0, "Use a multiple of three, so that it di #define DYNAMIC_FEE_REFERENCE_TRANSACTION_WEIGHT ((uint64_t)3000) #define DYNAMIC_FEE_REFERENCE_TRANSACTION_WEIGHT_V12 ((uint64_t)240000) // Only v12 (v13 switches back) +#define BLINK_MINER_FEE_MULTIPLE 1 // The blink fee that the miner including it earns (as a multiple of the base fee) +#define BLINK_TX_FEE_MULTIPLE 5 // The blink fee that the sender pays (the difference between this and MINER_FEE is burned) + +static_assert(BLINK_TX_FEE_MULTIPLE >= BLINK_MINER_FEE_MULTIPLE, "blink tx fees must be as least as large as blink miner fees"); +static_assert(BLINK_MINER_FEE_MULTIPLE >= 1, "blink miner fee cannot be smaller than the base tx fee"); + #define DIFFICULTY_TARGET_V2 120 // seconds #define DIFFICULTY_WINDOW_V2 60 #define DIFFICULTY_BLOCKS_COUNT_V2 (DIFFICULTY_WINDOW_V2 + 1) // added +1 to make N=N @@ -171,6 +177,7 @@ static_assert(STAKING_PORTIONS % 3 == 0, "Use a multiple of three, so that it di #define HF_VERSION_PER_OUTPUT_FEE cryptonote::network_version_13_enforce_checkpoints #define HF_VERSION_ED25519_KEY cryptonote::network_version_13_enforce_checkpoints #define HF_VERSION_FEE_BURNING cryptonote::network_version_14 +#define HF_VERSION_BLINK cryptonote::network_version_14 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 73b9bdaf4..82db27dbb 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -260,13 +260,15 @@ namespace cryptonote [[noreturn]] static void need_core_init() { throw std::logic_error("Internal error: quorumnet::init_core_callbacks() should have been called"); } - void *(*quorumnet_new)(core &, service_nodes::service_node_list &, const std::string &bind); - void (*quorumnet_delete)(void *self); + void *(*quorumnet_new)(core &, tx_memory_pool &, const std::string &bind); + void (*quorumnet_delete)(void *&self); void (*quorumnet_relay_votes)(void *self, const std::vector &); + std::future> (*quorumnet_send_blink)(void *self, const std::string &tx_blob); static bool init_core_callback_stubs() { - quorumnet_new = [](core &, service_nodes::service_node_list &, const std::string &) -> void * { need_core_init(); }; - quorumnet_delete = [](void *) { need_core_init(); }; + quorumnet_new = [](core &, tx_memory_pool &, const std::string &) -> void * { need_core_init(); }; + quorumnet_delete = [](void *&) { need_core_init(); }; quorumnet_relay_votes = [](void *, const std::vector &) { need_core_init(); }; + quorumnet_send_blink = [](void *, const std::string &) -> std::future> { need_core_init(); }; return false; } bool init_core_callback_complete = init_core_callback_stubs(); @@ -858,10 +860,15 @@ namespace cryptonote if (m_service_node_keys) { + std::lock_guard lock{m_quorumnet_init_mutex}; // quorumnet_new takes a zmq bind string, e.g. "tcp://1.2.3.4:5678" - std::string qnet_listen = "tcp://" + vm["p2p-bind-ip"].as() + ":" + std::to_string(m_quorumnet_port); - m_quorumnet_obj = quorumnet_new(*this, m_service_node_list, qnet_listen); + std::string listen_ip = vm["p2p-bind-ip"].as(); + if (listen_ip.empty()) + listen_ip = "0.0.0.0"; + std::string qnet_listen = "tcp://" + listen_ip + ":" + std::to_string(m_quorumnet_port); + m_quorumnet_obj = quorumnet_new(*this, m_mempool, qnet_listen); } + // Otherwise we may still need quorumnet in remote-only mode, but we construct it on demand return true; } @@ -958,10 +965,8 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::deinit() { - if (m_quorumnet_obj) { + if (m_quorumnet_obj) quorumnet_delete(m_quorumnet_obj); - m_quorumnet_obj = nullptr; - } m_service_node_list.store(); m_service_node_list.set_db_pointer(nullptr); m_miner.stop(); @@ -1244,6 +1249,17 @@ namespace cryptonote return r; } //----------------------------------------------------------------------------------------------- + std::future> core::handle_blink_tx(const std::string &tx_blob) + { + if (!m_quorumnet_obj) { + assert(!m_service_node_keys); + std::lock_guard lock{m_quorumnet_init_mutex}; + if (!m_quorumnet_obj) + m_quorumnet_obj = quorumnet_new(*this, m_mempool, "" /* don't listen */); + } + return quorumnet_send_blink(m_quorumnet_obj, tx_blob); + } + //----------------------------------------------------------------------------------------------- bool core::get_stat_info(core_stat_info& st_inf) const { st_inf.mining_speed = m_miner.get_speed(); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index aeede2540..8ff476d2c 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -31,6 +31,7 @@ #pragma once #include +#include #include #include @@ -70,17 +71,24 @@ namespace cryptonote extern const command_line::arg_descriptor arg_block_download_max_size; extern const command_line::arg_descriptor arg_recalculate_difficulty; + enum class blink_result { rejected, accepted, timeout }; + // Function pointers that are set to throwing stubs and get replaced by the actual functions in // cryptonote_protocol/quorumnet.cpp's quorumnet::init_core_callbacks(). This indirection is here // so that core doesn't need to link against cryptonote_protocol (plus everything it depends on). - // Starts the quorumnet listener. Return an opaque object (i.e. "this") that gets passed into all - // the other callbacks below. - extern void *(*quorumnet_new)(core &core, service_nodes::service_node_list &sn_list, const std::string &bind); - // Stops the quorumnet listener; is expected to delete the object. - extern void (*quorumnet_delete)(void *self); + // Starts the quorumnet listener. Return an opaque pointer (void *) that gets passed into all the + // other callbacks below so that the callbacks can recast it into whatever it should be. `bind` + // will be null if the quorumnet object is started in remote-only (non-listening) mode, which only + // happens on-demand when running in non-SN mode. + extern void *(*quorumnet_new)(core &core, tx_memory_pool &pool, const std::string &bind); + // Stops the quorumnet listener; is expected to delete the object and reset the pointer to nullptr. + extern void (*quorumnet_delete)(void *&self); // Relays votes via quorumnet. extern void (*quorumnet_relay_votes)(void *self, const std::vector &votes); + // Sends a blink tx to the current blink quorum, returns a future that can be used to wait for the + // result. + extern std::future> (*quorumnet_send_blink)(void *self, const std::string &tx_blob); extern bool init_core_callback_complete; @@ -106,7 +114,11 @@ namespace cryptonote * * @param pprotocol pre-constructed protocol object to store and use */ - core(i_cryptonote_protocol* pprotocol); + explicit core(i_cryptonote_protocol* pprotocol); + + // Non-copyable: + core(const core &) = delete; + core &operator=(const core &) = delete; /** * @copydoc Blockchain::handle_get_objects @@ -166,6 +178,18 @@ namespace cryptonote */ bool handle_incoming_txs(const std::vector& tx_blobs, std::vector& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + /** + * @brief handles an incoming blink transaction by dispatching it to the service node network + * via quorumnet. If this node is not a service node this will start up quorumnet in + * remote-only mode the first time it is called. + * + * @param tx_blob the transaction data + * + * @returns a pair of a blink result value: rejected, accepted, or timeout; and a rejection + * reason as returned by one of the blink quorum nodes. + */ + std::future> handle_blink_tx(const std::string &tx_blob); + /** * @brief handles an incoming block * @@ -1116,6 +1140,7 @@ namespace cryptonote std::string m_quorumnet_bind_ip; // Currently just copied from p2p-bind-ip void *m_quorumnet_obj = nullptr; + std::mutex m_quorumnet_init_mutex; size_t block_sync_size; diff --git a/src/cryptonote_core/service_node_list.cpp b/src/cryptonote_core/service_node_list.cpp index b41398af7..820e7d740 100644 --- a/src/cryptonote_core/service_node_list.cpp +++ b/src/cryptonote_core/service_node_list.cpp @@ -1269,7 +1269,7 @@ namespace service_nodes if (total_nodes >= BLINK_MIN_VOTES) { pub_keys_indexes = generate_shuffled_service_node_index_list(total_nodes, state.block_hash, type); - num_validators = std::min(pub_keys_indexes.size(), BLINK_MIN_VOTES); + num_validators = std::min(pub_keys_indexes.size(), BLINK_SUBQUORUM_SIZE); } // Otherwise leave empty to signal that there aren't enough SNs to form a usable quorum (to // distinguish it from an invalid height, which gets left as a nullptr) diff --git a/src/cryptonote_core/service_node_quorum_cop.cpp b/src/cryptonote_core/service_node_quorum_cop.cpp index 0b037cd97..f0cceab33 100644 --- a/src/cryptonote_core/service_node_quorum_cop.cpp +++ b/src/cryptonote_core/service_node_quorum_cop.cpp @@ -459,6 +459,9 @@ namespace service_nodes } } break; + + case quorum_type::blink: + break; } } } @@ -676,21 +679,19 @@ namespace service_nodes // then reading 1-8 from the second pubkey, 2-9 from the third, and so on, and adding all the // uint64_t values together. If we get to 25 we wrap the read around the end and keep going. uint64_t sum = 0; - uint64_t local; - auto *local_data = reinterpret_cast(&local); - offset %= KEY_BYTES; + alignas(uint64_t) std::array local; for (auto &pk : pubkeys) { + offset %= KEY_BYTES; auto *pkdata = reinterpret_cast(&pk); - if (offset <= KEY_BYTES - 8) - std::memcpy(local_data, pkdata + offset, 8); + if (offset <= KEY_BYTES - sizeof(uint64_t)) + std::memcpy(local.data(), pkdata + offset, sizeof(uint64_t)); else { size_t prewrap = KEY_BYTES - offset; - std::memcpy(local_data, pkdata + offset, prewrap); - std::memcpy(local_data + prewrap, pkdata, 8 - prewrap); + std::memcpy(local.data(), pkdata + offset, prewrap); + std::memcpy(local.data() + prewrap, pkdata, sizeof(uint64_t) - prewrap); } - boost::endian::little_to_native_inplace(local); - sum += local; - offset = (offset + 1) % KEY_BYTES; + sum += boost::endian::little_to_native(*reinterpret_cast(local.data())); + ++offset; } return sum; } diff --git a/src/cryptonote_core/service_node_rules.h b/src/cryptonote_core/service_node_rules.h index 7d985bae0..1c39f35e2 100644 --- a/src/cryptonote_core/service_node_rules.h +++ b/src/cryptonote_core/service_node_rules.h @@ -38,9 +38,9 @@ namespace service_nodes { "The maximum number of votes a service node can miss cannot be greater than the amount of checkpoint " "quorums they must participate in before we check if they should be deregistered or not."); - constexpr uint64_t BLINK_QUORUM_INTERVAL = 5; // We generate a new sub-quorum every N blocks (two consecutive quorums are needed for a blink signature) - constexpr uint64_t BLINK_QUORUM_LAG = 7 * BLINK_QUORUM_INTERVAL; // The lag (which must be a multiple of BLINK_QUORUM_INTERVAL) in determining the base blink quorum height - constexpr uint64_t BLINK_EXPIRY_BUFFER = BLINK_QUORUM_LAG + 10; // We don't select any SNs that have a scheduled unlock within this many blocks (measured from the lagged height) + constexpr int BLINK_QUORUM_INTERVAL = 5; // We generate a new sub-quorum every N blocks (two consecutive quorums are needed for a blink signature) + constexpr int BLINK_QUORUM_LAG = 7 * BLINK_QUORUM_INTERVAL; // The lag (which must be a multiple of BLINK_QUORUM_INTERVAL) in determining the base blink quorum height + constexpr int BLINK_EXPIRY_BUFFER = BLINK_QUORUM_LAG + 10; // We don't select any SNs that have a scheduled unlock within this many blocks (measured from the lagged height) static_assert(BLINK_QUORUM_LAG % BLINK_QUORUM_INTERVAL == 0, "BLINK_QUORUM_LAG must be an integral multiple of BLINK_QUORUM_INTERVAL"); static_assert(BLINK_EXPIRY_BUFFER > BLINK_QUORUM_LAG + BLINK_QUORUM_INTERVAL, "BLINK_EXPIRY_BUFFER is too short to cover a blink quorum height range"); @@ -56,16 +56,16 @@ namespace service_nodes { constexpr int MIN_TIME_IN_S_BEFORE_VOTING = 0; constexpr size_t CHECKPOINT_QUORUM_SIZE = 5; constexpr size_t CHECKPOINT_MIN_VOTES = 1; - constexpr size_t BLINK_SUBQUORUM_SIZE = 5; - constexpr size_t BLINK_MIN_VOTES = 1; + constexpr int BLINK_SUBQUORUM_SIZE = 5; + constexpr int BLINK_MIN_VOTES = 1; #else constexpr size_t STATE_CHANGE_MIN_VOTES_TO_CHANGE_STATE = 7; constexpr size_t STATE_CHANGE_QUORUM_SIZE = 10; constexpr int MIN_TIME_IN_S_BEFORE_VOTING = UPTIME_PROOF_MAX_TIME_IN_SECONDS; constexpr size_t CHECKPOINT_QUORUM_SIZE = 20; constexpr size_t CHECKPOINT_MIN_VOTES = 13; - constexpr size_t BLINK_SUBQUORUM_SIZE = 10; - constexpr size_t BLINK_MIN_VOTES = 7; + constexpr int BLINK_SUBQUORUM_SIZE = 10; + constexpr int BLINK_MIN_VOTES = 7; #endif static_assert(STATE_CHANGE_MIN_VOTES_TO_CHANGE_STATE <= STATE_CHANGE_QUORUM_SIZE, "The number of votes required to kick can't exceed the actual quorum size, otherwise we never kick."); @@ -124,15 +124,15 @@ namespace service_nodes { std::numeric_limits::max(); }; - inline quorum_type max_quorum_type_for_hf(uint8_t hf_version) + constexpr quorum_type max_quorum_type_for_hf(uint8_t hf_version) { - quorum_type result = (hf_version <= cryptonote::network_version_12_checkpointing) ? quorum_type::obligations - : quorum_type::checkpointing; - assert(result != quorum_type::_count); - return result; + return + hf_version <= cryptonote::network_version_12_checkpointing ? quorum_type::obligations : + hf_version < cryptonote::network_version_14 ? quorum_type::checkpointing : + quorum_type::blink; } - inline uint64_t staking_num_lock_blocks(cryptonote::network_type nettype) + constexpr uint64_t staking_num_lock_blocks(cryptonote::network_type nettype) { switch (nettype) { diff --git a/src/cryptonote_core/tx_blink.cpp b/src/cryptonote_core/tx_blink.cpp index 37665a06c..29892883c 100644 --- a/src/cryptonote_core/tx_blink.cpp +++ b/src/cryptonote_core/tx_blink.cpp @@ -1,67 +1,113 @@ +// Copyright (c) 2019, The Loki Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + #include "tx_blink.h" #include "common/util.h" #include "service_node_list.h" #include +#include "../cryptonote_basic/cryptonote_format_utils.h" namespace cryptonote { using namespace service_nodes; -static void check_args(blink_tx::subquorum q, unsigned position, const char *func_name) { - if (q >= blink_tx::subquorum::_count) - throw std::domain_error("Invalid sub-quorum value passed to " + std::string(func_name)); - if (position >= BLINK_SUBQUORUM_SIZE) - throw std::domain_error("Invalid voter position passed to " + std::string(func_name)); +static void check_args(blink_tx::subquorum q, int position, const char *func_name) { + if (q < blink_tx::subquorum::base || q >= blink_tx::subquorum::_count) + throw std::invalid_argument("Invalid sub-quorum value passed to " + std::string(func_name)); + if (position < 0 || position >= BLINK_SUBQUORUM_SIZE) + throw std::invalid_argument("Invalid voter position passed to " + std::string(func_name)); } -crypto::public_key blink_tx::get_sn_pubkey(subquorum q, unsigned position, const service_node_list &snl) const { - check_args(q, position, __func__); - uint64_t qheight = quorum_height(q); - auto blink_quorum = snl.get_quorum(quorum_type::blink, qheight); - if (!blink_quorum) { - // TODO FIXME XXX - we don't want a failure here; if this happens we need to go back into state - // history to retrieve the state info. - MERROR("FIXME: could not get blink quorum for blink_tx"); +crypto::public_key blink_tx::get_sn_pubkey(subquorum q, int position, const service_node_list &snl) const { + check_args(q, position, __func__); + uint64_t qheight = quorum_height(q); + auto blink_quorum = snl.get_quorum(quorum_type::blink, qheight); + if (!blink_quorum) { + // TODO FIXME XXX - we don't want a failure here; if this happens we need to go back into state + // history to retrieve the state info. + MERROR("FIXME: could not get blink quorum for blink_tx"); + return crypto::null_pkey; + } + + if (position < (int) blink_quorum->validators.size()) + return blink_quorum->validators[position]; + return crypto::null_pkey; - } - - if (position < blink_quorum->validators.size()) - return blink_quorum->validators[position]; - - return crypto::null_pkey; }; -crypto::hash blink_tx::hash() const { - auto buf = tools::memcpy_le(height_, tx_->hash); - crypto::hash hash; - crypto::cn_fast_hash(buf.data(), buf.size(), hash); - return hash; +crypto::hash blink_tx::hash(bool approved) const { + crypto::hash tx_hash, blink_hash; + if (!cryptonote::get_transaction_hash(tx, tx_hash)) + throw std::runtime_error("Cannot compute blink hash: tx hash is not valid"); + auto buf = tools::memcpy_le(height, tx_hash, uint8_t{approved}); + crypto::cn_fast_hash(buf.data(), buf.size(), blink_hash); + return blink_hash; } -bool blink_tx::add_signature(subquorum q, unsigned position, const crypto::signature &sig, const service_node_list &snl) { - check_args(q, position, __func__); - auto &sig_slot = signatures_[static_cast(q)][position]; - if (sig_slot && sig_slot == sig) - return false; +bool blink_tx::add_signature(subquorum q, int position, bool approved, const crypto::signature &sig, const service_node_list &snl) { + check_args(q, position, __func__); - if (!crypto::check_signature(hash(), get_sn_pubkey(q, position, snl), sig)) - throw signature_verification_error("Given blink quorum signature verification failed!"); + if (!crypto::check_signature(hash(approved), get_sn_pubkey(q, position, snl), sig)) + throw signature_verification_error("Given blink quorum signature verification failed!"); - sig_slot = sig; - return true; + return add_prechecked_signature(q, position, approved, sig); } -bool blink_tx::has_signature(subquorum q, unsigned position) { - check_args(q, position, __func__); - return signatures_[static_cast(q)][position]; +bool blink_tx::add_prechecked_signature(subquorum q, int position, bool approved, const crypto::signature &sig) { + check_args(q, position, __func__); + + auto &sig_slot = signatures_[static_cast(q)][position]; + if (sig_slot.status != signature_status::none) + return false; + + sig_slot.status = approved ? signature_status::approved : signature_status::rejected; + sig_slot.sig = sig; + return true; } -bool blink_tx::valid() const { - // Signatures are verified when added, so here we can just test that they are non-null - return std::all_of(signatures_.begin(), signatures_.end(), [](const auto &sigs) { - return std::count_if(sigs.begin(), sigs.end(), [](const auto &s) -> bool { return s; }) >= int{BLINK_MIN_VOTES}; - }); +blink_tx::signature_status blink_tx::get_signature_status(subquorum q, int position) const { + check_args(q, position, __func__); + return signatures_[static_cast(q)][position].status; +} + +bool blink_tx::approved() const { + return std::all_of(signatures_.begin(), signatures_.end(), [](const auto &sigs) { + return std::count_if(sigs.begin(), sigs.end(), [](const quorum_signature &s) -> bool { return s.status == signature_status::approved; }) + >= BLINK_MIN_VOTES; + }); +} + +bool blink_tx::rejected() const { + return std::any_of(signatures_.begin(), signatures_.end(), [](const auto &sigs) { + return std::count_if(sigs.begin(), sigs.end(), [](const quorum_signature &s) -> bool { return s.status == signature_status::rejected; }) + > BLINK_SUBQUORUM_SIZE - BLINK_MIN_VOTES; + }); } } diff --git a/src/cryptonote_core/tx_blink.h b/src/cryptonote_core/tx_blink.h index 593e5f47f..e8957409b 100644 --- a/src/cryptonote_core/tx_blink.h +++ b/src/cryptonote_core/tx_blink.h @@ -32,6 +32,7 @@ #include "../common/util.h" #include "service_node_rules.h" #include +#include namespace service_nodes { class service_node_list; @@ -43,81 +44,107 @@ namespace cryptonote { class blink_tx { public: - enum class subquorum : uint8_t { base, future, _count }; + enum class subquorum : uint8_t { base, future, _count }; - class signature_verification_error : public std::runtime_error { - using std::runtime_error::runtime_error; - }; + enum class signature_status : uint8_t { none, rejected, approved }; - /** - * Construct a new blink_tx wrapper given the tx and a blink authorization height. - */ - blink_tx(std::shared_ptr tx, uint64_t height) - : tx_{std::move(tx)}, height_{height} { - signatures_.fill({}); - } + /// The blink authorization height of this blink tx, i.e. the block height at the time the + /// transaction was created. + const uint64_t height; - /** Construct a new blink_tx from just a height; constructs a default transaction. - */ - explicit blink_tx(uint64_t height) : blink_tx(std::make_shared(), height) {} + /// The encapsulated transaction. Any modifications to this (including getting the tx_hash, + /// which gets cached!) should either take place before the object is shared, or protected by + /// the unique_lock. + transaction tx; - /** - * Adds a signature for the given quorum and position. Returns true if the signature was accepted - * (i.e. is valid, and existing signature is empty), false if the signature was already present, - * and throws a `blink_tx::signature_verification_error` if the signature fails validation. - */ - bool add_signature(subquorum q, unsigned int position, const crypto::signature &sig, const service_nodes::service_node_list &snl); + class signature_verification_error : public std::runtime_error { + using std::runtime_error::runtime_error; + }; - /** - * Remove the signature at the given quorum and position by setting it to null. Returns true if - * removed, false if it was already null. - */ - bool clear_signature(subquorum q, unsigned int position); + // Not default constructible + blink_tx() = delete; - /** - * Returns true if there is a verified signature at the given quorum and position. - */ - bool has_signature(subquorum q, unsigned int position); + /** Construct a new blink_tx from just a height; constructs a default transaction. + */ + explicit blink_tx(uint64_t height) : tx{}, height{height} { + assert(quorum_height(subquorum::base) > 0); + for (auto &q : signatures_) + for (auto &s : q) + s.status = signature_status::none; + } - /** - * Returns true if this blink tx is valid for inclusion in the blockchain, that is, has the - * required number of valid signatures in each quorum. - */ - bool valid() const; + /// Obtains a unique lock on this blink tx; required for any signature-mutating method unless + /// otherwise noted + template + auto unique_lock(Args &&...args) { return std::unique_lock{mutex_, std::forward(args)...}; } - /// Returns a reference to the transaction. - transaction &tx() { return *tx_; } + /// Obtains a unique lock on this blink tx; required for any signature-dependent method unless + /// otherwise noted + template + auto shared_lock(Args &&...args) { return std::shared_lock{mutex_, std::forward(args)...}; } - /// Returns a reference to the transaction, const version. - const transaction &tx() const { return *tx_; } + /** + * Adds a signature for the given quorum and position. Returns false if a signature was already + * present; true if the signature was accepted and stored; and throws a + * `blink_tx::signature_verification_error` if the signature fails validation. + */ + bool add_signature(subquorum q, int position, bool approved, const crypto::signature &sig, const service_nodes::service_node_list &snl); - /// Returns the blink authorization height of this blink tx, i.e. the block height at the time the - /// transaction was created. - uint64_t height() const { return height_; } + /** + * Adds a signature for the given quorum and position without checking it for validity (i.e. + * because it has already been checked with crypto::check_signature). Returns true if added, + * false if a signature was already present. + */ + bool add_prechecked_signature(subquorum q, int position, bool approved, const crypto::signature &sig); - /// Returns the quorum height for the given height and quorum (base or future); returns 0 at the - /// beginning of the chain (before there are enough blocks for a blink quorum). - static uint64_t quorum_height(uint64_t h, subquorum q) { - uint64_t bh = h - (h % service_nodes::BLINK_QUORUM_INTERVAL) - service_nodes::BLINK_QUORUM_LAG - + static_cast(q) * service_nodes::BLINK_QUORUM_INTERVAL; - return bh > h /*overflow*/ ? 0 : bh; - } + /** + * Returns the signature status for the given subquorum and position. + */ + signature_status get_signature_status(subquorum q, int position) const; - /// Returns the quorum height for the given quorum (base or future); returns 0 at the beginning of - /// the chain (before there are enough blocks for a blink quorum). - uint64_t quorum_height(subquorum q) const { return quorum_height(height_, q); } + /** + * Returns true if this blink tx is valid for inclusion in the blockchain, that is, has the + * required number of approval signatures in each quorum. (Note that it is possible for a blink + * tx to be neither approved() nor rejected()). You must hold a shared_lock when calling this. + */ + bool approved() const; - /// Returns the pubkey of the referenced service node, or null if there is no such service node. - crypto::public_key get_sn_pubkey(subquorum q, unsigned position, const service_nodes::service_node_list &snl) const; + /** + * Returns true if this blink tx has been definitively rejected, that is, has enough rejection + * signatures in at least one of the quorums that it is impossible for it to become approved(). + * (Note that it is possible for a blink tx to be neither approved() nor rejected()). You must + * hold a shared_lock when calling this. + */ + bool rejected() const; - /// Returns the hashed signing value for this blink TX (a fast hash of the height + tx hash) - crypto::hash hash() const; + /// Returns the quorum height for the given height and quorum (base or future); returns 0 at the + /// beginning of the chain (before there are enough blocks for a blink quorum). + static uint64_t quorum_height(uint64_t h, subquorum q) { + uint64_t bh = h - (h % service_nodes::BLINK_QUORUM_INTERVAL) - service_nodes::BLINK_QUORUM_LAG + + static_cast(q) * service_nodes::BLINK_QUORUM_INTERVAL; + return bh > h /*overflow*/ ? 0 : bh; + } + + /// Returns the height of the given subquorum (base or future) for this blink tx; returns 0 at + /// the beginning of the chain (before there are enough blocks for a blink quorum). Lock not + /// required. + uint64_t quorum_height(subquorum q) const { return quorum_height(height, q); } + + /// Returns the pubkey of the referenced service node, or null if there is no such service node. + crypto::public_key get_sn_pubkey(subquorum q, int position, const service_nodes::service_node_list &snl) const; + + /// Returns the hashed signing value for this blink TX for a tx with status `approved`. The + /// result is a fast hash of the height + tx hash + approval value. Lock not required. + crypto::hash hash(bool approved) const; + + struct quorum_signature { + signature_status status; + crypto::signature sig; + }; private: - std::shared_ptr tx_; - uint64_t height_; - std::array, tools::enum_count> signatures_; - + std::array, tools::enum_count> signatures_; + std::shared_timed_mutex mutex_; }; } diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index ee4ad8265..fffcb8272 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -114,7 +114,7 @@ namespace cryptonote }; } //--------------------------------------------------------------------------------- - //--------------------------------------------------------------------------------- + // warning: bchs is passed here uninitialized, so don't do anything but store it tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_txpool_max_weight(DEFAULT_TXPOOL_MAX_WEIGHT), m_txpool_weight(0), m_cookie(0) { @@ -223,6 +223,12 @@ namespace cryptonote return false; } + + // Blink notes: a blink quorum member adds an incoming blink tx into the mempool to make sure it + // can be accepted, but sets it as do_not_relay initially. If it gets added, the quorum member + // sends a signature to other quorum members. Once enough signatures are received it updates it + // to set `do_not_relay` to false and starts relaying it (other quorum members do the same). + //--------------------------------------------------------------------------------- bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) { @@ -430,6 +436,35 @@ namespace cryptonote return add_tx(tx, h, bl, get_transaction_weight(tx, bl.size()), tvc, keeped_by_block, relayed, do_not_relay, version); } //--------------------------------------------------------------------------------- + bool tx_memory_pool::add_blink(const std::shared_ptr &blink_ptr, tx_verification_context &tvc, bool &blink_exists) + { + assert((bool) blink_ptr); + std::unique_lock lock(m_blinks_mutex); + CRITICAL_REGION_LOCAL(m_transactions_lock); + auto &tx = blink_ptr->tx; + auto txhash = get_transaction_hash(tx); + auto &ptr = m_blinks[txhash]; + blink_exists = (bool) ptr; + if (blink_exists) + return false; + ptr = blink_ptr; + auto &blink = *ptr; + auto hf_version = m_blockchain.get_ideal_hard_fork_version(blink.height); + bool ret = add_tx(tx, tvc, false /*kept_by_block*/, false /*relayed*/, true /*do_not_relay*/, hf_version); + if (!ret) + m_blinks.erase(txhash); + return ret; + } + //--------------------------------------------------------------------------------- + std::shared_ptr tx_memory_pool::get_blink(const crypto::hash &tx_hash) const + { + std::shared_lock lock(m_blinks_mutex); + auto it = m_blinks.find(tx_hash); + if (it != m_blinks.end()) + return it->second; + return {}; + } + //--------------------------------------------------------------------------------- size_t tx_memory_pool::get_txpool_weight() const { CRITICAL_REGION_LOCAL(m_transactions_lock); diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 980c934b6..231ffb820 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -46,11 +46,7 @@ #include "crypto/hash.h" #include "rpc/core_rpc_server_commands_defs.h" #include "rpc/message_data_structs.h" - -namespace service_nodes -{ - class service_node_list; -}; +#include "tx_blink.h" namespace cryptonote { @@ -92,7 +88,7 @@ namespace cryptonote * helping create a new block template by choosing transactions for it * */ - class tx_memory_pool: boost::noncopyable + class tx_memory_pool { public: /** @@ -102,6 +98,9 @@ namespace cryptonote */ tx_memory_pool(Blockchain& bchs); + // Non-copyable + tx_memory_pool(const tx_memory_pool &) = delete; + tx_memory_pool &operator=(const tx_memory_pool &) = delete; /** * @copydoc add_tx(transaction&, tx_verification_context&, bool, bool, uint8_t) @@ -130,6 +129,25 @@ namespace cryptonote */ bool add_tx(transaction &tx, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); + /** + * @brief attempts to add a blink transaction to the transaction pool and blink pool. The + * transaction is set for relaying if it has the required blink signatures, and not relayed + * otherwise. + * + * @param blink - a shared_ptr to the blink details + * @param tvc - the verification results + * @param blink_exists - will be set to true if the addition fails because the blink tx already + * exists + * + * @return true if the tx passes validations and has been added to tx/blink pools + */ + bool add_blink(const std::shared_ptr &blink, tx_verification_context& tvc, bool &blink_exists); + + /** + * @brief accesses blink tx details if the given tx hash is a known blink tx, nullptr otherwise. + */ + std::shared_ptr get_blink(const crypto::hash &tx_hash) const; + /** * @brief takes a transaction with the given hash from the pool * @@ -587,6 +605,11 @@ namespace cryptonote mutable std::unordered_map> m_input_cache; std::unordered_map m_parsed_tx_cache; + + mutable std::shared_timed_mutex m_blinks_mutex; + // { height => { txhash => blink_tx, ... }, ... } + std::unordered_map> m_blinks; + // TODO: clean up m_blinks once mined & immutably checkpointed }; } diff --git a/src/cryptonote_protocol/quorumnet.cpp b/src/cryptonote_protocol/quorumnet.cpp index 8fb429450..1cb6856e0 100644 --- a/src/cryptonote_protocol/quorumnet.cpp +++ b/src/cryptonote_protocol/quorumnet.cpp @@ -31,35 +31,70 @@ #include "cryptonote_core/service_node_voting.h" #include "cryptonote_core/service_node_rules.h" #include "cryptonote_core/tx_blink.h" +#include "cryptonote_core/tx_pool.h" #include "quorumnet/sn_network.h" #include "quorumnet/conn_matrix.h" #include "cryptonote_config.h" +#include + #undef LOKI_DEFAULT_LOG_CATEGORY #define LOKI_DEFAULT_LOG_CATEGORY "qnet" +namespace quorumnet { + namespace { -using namespace quorumnet; using namespace service_nodes; +using namespace std::string_literals; +using namespace std::chrono_literals; + +using blink_tx = cryptonote::blink_tx; + +constexpr auto NUM_BLINK_QUORUMS = tools::enum_count; +static_assert(std::is_same(), "unexpected underlying blink quorum count type"); + +using quorum_array = std::array, NUM_BLINK_QUORUMS>; + +using pending_signature = std::tuple; // approval, subquorum, subquorum position, signature + +struct pending_signature_hash { + size_t operator()(const pending_signature &s) const { return std::get(s) + std::hash{}(std::get(s)); } +}; + +using pending_signature_set = std::unordered_set; struct SNNWrapper { SNNetwork snn; - cryptonote::core &core; // FIXME - may not be needed? Can we get everything needed via sn_list? - service_node_list &sn_list; + cryptonote::core &core; + cryptonote::tx_memory_pool &pool; - std::mutex blinks_mutex; - // { height => { txhash => blink_tx, ... }, ... } - std::map>> blinks; + // Track submitted blink txes here; unlike the blinks stored in the mempool we store these ones + // more liberally to track submitted blinks, even if unsigned/unacceptable, while the mempool + // only stores approved blinks. + std::shared_timed_mutex mutex; + + struct blink_metadata { + std::shared_ptr btxptr; + pending_signature_set pending_sigs; + std::string reply_pubkey; + uint64_t reply_tag = 0; + }; + // { height => { txhash => {blink_tx,sigs,reply}, ... }, ... } + std::map> blinks; + + // FIXME: + //std::chrono::steady_clock::time_point last_blink_cleanup = std::chrono::steady_clock::now(); template - SNNWrapper(cryptonote::core &core, service_node_list &sn_list, Args &&...args) : - snn{std::forward(args)...}, core{core}, sn_list{sn_list} {} + SNNWrapper(cryptonote::core &core, cryptonote::tx_memory_pool &pool, Args &&...args) : + snn{std::forward(args)...}, core{core}, pool{pool} {} }; template -std::string key_data_as_string(const T &key) { - return {reinterpret_cast(key.data), sizeof(key.data)}; +std::string get_data_as_string(const T &key) { + static_assert(std::is_trivial(), "cannot safely copy non-trivial class to string"); + return {reinterpret_cast(&key), sizeof(key)}; } crypto::x25519_public_key x25519_from_string(const std::string &pubkey) { @@ -110,32 +145,56 @@ void snn_write_log(LogLevel level, const char *file, int line, std::string msg) el::base::Writer(easylogging_level(level), file, line, ELPP_FUNC, el::base::DispatchAction::NormalLog).construct(LOKI_DEFAULT_LOG_CATEGORY) << msg; } -void *new_snnwrapper(cryptonote::core &core, service_node_list &sn_list, const std::string &bind) { +void *new_snnwrapper(cryptonote::core &core, cryptonote::tx_memory_pool &pool, const std::string &bind) { auto keys = core.get_service_node_keys(); - assert((bool) keys); - MINFO("Starting quorumnet listener on " << bind << " with x25519 pubkey " << keys->pub_x25519); - auto *obj = new SNNWrapper(core, sn_list, - key_data_as_string(keys->pub_x25519), - key_data_as_string(keys->key_x25519), + auto peer_lookup = [&sn_list = core.get_service_node_list()](const std::string &x25519_pub) { + return get_connect_string(sn_list, x25519_from_string(x25519_pub)); + }; + auto allow = [&sn_list = core.get_service_node_list()](const std::string &ip, const std::string &x25519_pubkey_str) { + auto x25519_pubkey = x25519_from_string(x25519_pubkey_str); + auto pubkey = sn_list.get_pubkey_from_x25519(x25519_pubkey); + if (pubkey) { + MINFO("Accepting incoming SN connection authentication from ip/x25519/pubkey: " << ip << "/" << x25519_pubkey << "/" << pubkey); + return SNNetwork::allow::service_node; + } + + // Public connection: + // + // TODO: we really only want to accept public connections here if we are in (or soon + // to be or recently were in) a blink quorum; at other times we want to refuse a + // non-SN connection. We could also IP limit throttle. + // + // (In theory we could extend this to also only allow SN + // connections when in or near a blink/checkpoint/obligations/pulse quorum, but that + // would get messy fast and probably have little practical benefit). + return SNNetwork::allow::client; + }; + SNNWrapper *obj; + if (!keys) { + MINFO("Starting remote-only quorumnet instance"); + + obj = new SNNWrapper(core, pool, peer_lookup, allow, snn_want_log, snn_write_log); + } else { + MINFO("Starting quorumnet listener on " << bind << " with x25519 pubkey " << keys->pub_x25519); + obj = new SNNWrapper(core, pool, + get_data_as_string(keys->pub_x25519), + get_data_as_string(keys->key_x25519.data), std::vector{{bind}}, - [&sn_list](const std::string &x25519_pub) { return get_connect_string(sn_list, x25519_from_string(x25519_pub)); }, - [&sn_list](const std::string &ip, const std::string &x25519_pubkey_str) { - // TODO: this function could also check to see whether the given pubkey *should* be - // contacting us (i.e. either will soon be or was recently in a shared quorum). - return true; - return (bool) sn_list.get_pubkey_from_x25519(x25519_from_string(x25519_pubkey_str)); - }, + peer_lookup, + allow, snn_want_log, snn_write_log); + } obj->snn.data = obj; // Provide pointer to the instance for callbacks return obj; } -void delete_snnwrapper(void *obj) { +void delete_snnwrapper(void *&obj) { auto *snn = reinterpret_cast(obj); MINFO("Shutting down quorumnet listener"); delete snn; + obj = nullptr; } @@ -147,6 +206,225 @@ E get_enum(const bt_dict &d, const std::string &key) { throw std::invalid_argument("invalid enum value for field " + key); } +/// Helper class to calculate and relay to peers of quorums. +/// +/// TODO: add a wrapper that caches this so that looking up the same quorum peers within a certain +/// amount of time doesn't need to recalculate. +class peer_info { +public: + using exclude_set = std::unordered_set; + + /// Maps pubkeys to x25519 pubkeys and zmq connection strings + std::unordered_map> remotes; + /// Stores the x25519 string pubkeys to either zmq connection strings (for a "strong" + /// connection) or empty strings (for an opportunistic "weak" connection). + std::unordered_map peers; + /// The number of strong peers, that is, the count of `peers` that has a non-empty second value. + /// Will be the same as `peers.count()` if opportunistic connections are disabled. + int strong_peers; + /// The caller's positions in the given quorum(s), -1 if not found + std::vector my_position; + /// The number of actual positions found in my_position (i.e. the number of elements of + /// `my_position` not equal to -1). + int my_position_count; + + /// Singleton wrapper around peer_info + peer_info( + SNNWrapper &snw, + quorum_type q_type, + std::shared_ptr &quorum, + bool opportunistic = true, + exclude_set exclude = {} + ) + : peer_info(snw, q_type, &quorum, &quorum + 1, opportunistic, std::move(exclude)) {} + + /// Constructs peer information for the given quorums and quorum position of the caller. + /// \param snw - the SNNWrapper reference + /// \param q_type - the type of quorum + /// \param qbegin, qend - the iterators to a set of pointers (or other deferenceable type) to quorums + /// \param opportunistic - if true then the peers to relay will also attempt to relay to any + /// incoming peers *if* those peers are already connected when the message is relayed. + /// \param exclude - can be specified as a set of peers that should be excluded from the peer + /// list. Typically for peers that we already know have the relayed information. This SN's + /// pubkey is always added to this exclude list. + template + peer_info( + SNNWrapper &snw, + quorum_type q_type, + QuorumIt qbegin, QuorumIt qend, + bool opportunistic = true, + std::unordered_set exclude = {} + ) + : snn{snw.snn} { + + auto keys = snw.core.get_service_node_keys(); + assert(keys); + const auto &my_pubkey = keys->pub; + exclude.insert(my_pubkey); + + // Find my positions in the quorums + my_position_count = 0; + for (auto qit = qbegin; qit != qend; ++qit) { + auto &v = (*qit)->validators; + auto found = std::find(v.begin(), v.end(), my_pubkey); + if (found == v.end()) + my_position.push_back(-1); + else { + my_position.push_back(std::distance(v.begin(), found)); + my_position_count++; + } + } + + std::unordered_set need_remotes; + auto qit = qbegin; + // Figure out all the remotes we need to be able to lookup (so that we can do all lookups in + // a single shot -- since it requires a mutex). + for (size_t i = 0; qit != qend; ++i, ++qit) { + const auto &v = (*qit)->validators; + for (int j : quorum_outgoing_conns(my_position[i], v.size())) + if (!exclude.count(v[j])) + need_remotes.insert(v[j]); + if (opportunistic) + for (int j : quorum_incoming_conns(my_position[i], v.size())) + if (!exclude.count(v[j])) + need_remotes.insert(v[j]); + } + + // Lookup the x25519 and ZMQ connection string for all peers + snw.core.get_service_node_list().for_each_service_node_info(need_remotes.begin(), need_remotes.end(), + [this](const crypto::public_key &pubkey, const service_nodes::service_node_info &info) { + if (!info.is_active()) return; + auto &proof = *info.proof; + if (!proof.pubkey_x25519 || !proof.quorumnet_port || !proof.public_ip) return; + remotes.emplace(pubkey, + std::make_pair(proof.pubkey_x25519, "tcp://" + epee::string_tools::get_ip_string_from_int32(proof.public_ip) + ":" + std::to_string(proof.quorumnet_port))); + }); + + compute_peers(qbegin, qend, opportunistic); + } + + /// Relays a command and any number of serialized data to everyone we're supposed to relay to + template + void relay_to_peers(const std::string &cmd, const T &...data) { + relay_to_peers_impl(cmd, std::array{send_option::serialized{data}...}, + std::make_index_sequence{}); + } + +private: + SNNetwork &snn; + + /// Looks up a pubkey in known remotes and adds it to `peers`. If strong, it is added with an + /// address, otherwise it is added with an empty address. If the element already exists, it + /// will be updated *if* it the existing entry is weak and `strong` is true, otherwise it will + /// be left as is. Returns true if a new entry was created or a weak entry was upgraded. + bool add_peer(const crypto::public_key &pubkey, bool strong = true) { + auto it = remotes.find(pubkey); + if (it != remotes.end()) { + std::string remote_addr = strong ? it->second.second : ""s; + auto ins = peers.emplace(get_data_as_string(it->second.first), std::move(remote_addr)); + if (strong && !ins.second && ins.first->second.empty()) { + ins.first->second = it->second.second; + strong_peers++; + return true; // Upgraded weak to strong + } + if (strong && ins.second) + strong_peers++; + + return ins.second; + } + return false; + } + + // Build a map of x25519 keys -> connection strings of all our quorum peers we talk to; the + // connection string is non-empty only for *strong* peer (i.e. one we should connect to if not + // already connected) and empty if it's an opportunistic peer (i.e. only send along if we already + // have a connection). + template + void compute_peers(QuorumIt qbegin, QuorumIt qend, bool opportunistic) { + + // TODO: when we receive a new block, if our quorum starts soon we can tell SNNetwork to + // pre-connect (to save the time in handshaking when we get an actual blink tx). + + strong_peers = 0; + + size_t i = 0; + for (QuorumIt qit = qbegin; qit != qend; ++i, ++qit) { + if (my_position[i] < 0) { + MTRACE("Not in subquorum " << (i == 0 ? "Q" : "Q'")); + continue; + } + + auto &validators = (*qit)->validators; + + // Relay to all my outgoing targets within the quorum (connecting if not already connected) + for (int j : quorum_outgoing_conns(my_position[i], validators.size())) { + if (add_peer(validators[j])) + MTRACE("Relaying within subquorum " << (i == 0 ? "Q" : "Q'") << " to service node " << validators[j]); + } + + // Opportunistically relay to all my *incoming* sources within the quorum *if* I already + // have a connection open with them, but don't open a new connection if I don't. + for (int j : quorum_incoming_conns(my_position[i], validators.size())) { + if (add_peer(validators[j], false /*!strong*/)) + MTRACE("Optional opportunistic relay within quorum " << (i == 0 ? "Q" : "Q'") << " to service node " << validators[j]); + } + + // Now establish strong interconnections between quorums, if we have multiple subquorums + // (i.e. blink quorums). + // + // If I'm in the last half* of the first quorum then I relay to the first half (roughly) of + // the next quorum. i.e. nodes 5-9 in Q send to nodes 0-4 in Q'. For odd numbers the last + // position gets left out (e.g. for 9 members total we would have 0-3 talk to 4-7 and no one + // talks to 8). + // + // (* - half here means half the size of the smaller quorum) + // + // We also skip this entirely if this SN is in both quorums since then we're already + // relaying to nodes in the next quorum. (Ideally we'd do the same if the recipient is in + // both quorums, but that's harder to figure out and so the special case isn't worth + // worrying about). + QuorumIt qnext = std::next(qit); + if (qnext != qend && my_position[i + 1] < 0) { + auto &next_validators = (*qnext)->validators; + int half = std::min(validators.size(), next_validators.size()) / 2; + if (my_position[i] >= half && my_position[i] < half*2) { + if (add_peer(validators[my_position[i] - half])) + MTRACE("Inter-quorum relay from Q to Q' service node " << next_validators[my_position[i] - half]); + } else { + MTRACE("Not a Q -> Q' inter-quorum relay (Q position is " << my_position[i] << ")"); + } + + } + + // Exactly the same connections as above, but in reverse and weak: the first half of Q' + // sends to the second half of Q. Typically this will end up reusing an already open + // connection, but if there isn't such an open connection then we establish a new one. + if (qit != qbegin && my_position[i - 1] < 0) { + auto &prev_validators = (*std::prev(qit))->validators; + int half = std::min(validators.size(), prev_validators.size()) / 2; + if (my_position[i] < half) { + if (add_peer(prev_validators[half + my_position[i]])) + MTRACE("Inter-quorum relay from Q' to Q service node " << prev_validators[my_position[i] - half]); + } else { + MTRACE("Not a Q' -> Q inter-quorum relay (Q' position is " << my_position[i] << ")"); + } + } + } + } + + /// Relays a command and pre-serialized data to everyone we're supposed to relay to + template + void relay_to_peers_impl(const std::string &cmd, std::array relay_data, std::index_sequence) { + for (auto &peer : peers) { + MTRACE("Relaying " << cmd << " to peer " << as_hex(peer.first) << (peer.second.empty() ? " (if connected)"s : " @ " + peer.second)); + if (peer.second.empty()) + snn.send(peer.first, cmd, relay_data[I]..., send_option::optional{}); + else + snn.send(peer.first, cmd, relay_data[I]..., send_option::hint{peer.second}); + } + } + +}; bt_dict serialize_vote(const quorum_vote_t &vote) { @@ -156,7 +434,7 @@ bt_dict serialize_vote(const quorum_vote_t &vote) { {"h", vote.block_height}, {"g", static_cast(vote.group)}, {"i", vote.index_in_group}, - {"s", std::string{reinterpret_cast(&vote.signature), sizeof(vote.signature)}}, + {"s", get_data_as_string(vote.signature)}, }; if (vote.type == quorum_type::checkpointing) result["bh"] = std::string{vote.checkpoint.block_hash.data, sizeof(crypto::hash)}; @@ -191,22 +469,7 @@ quorum_vote_t deserialize_vote(const bt_value &v) { return vote; } -/// Returns primary_pubkey => {x25519_pubkey, connect_string} pairs for the given primary pubkeys -template -static auto get_zmq_remotes(SNNWrapper &snw, It begin, It end) { - std::unordered_map> remotes; - snw.sn_list.for_each_service_node_info(begin, end, - [&remotes](const crypto::public_key &pubkey, const service_nodes::service_node_info &info) { - if (!info.is_active()) return; - auto &proof = *info.proof; - if (!proof.pubkey_x25519 || !proof.quorumnet_port || !proof.public_ip) return; - remotes.emplace(pubkey, - std::make_pair(proof.pubkey_x25519, "tcp://" + epee::string_tools::get_ip_string_from_int32(proof.public_ip) + ":" + std::to_string(proof.quorumnet_port))); - }); - return remotes; -} - -static void relay_votes(void *obj, const std::vector &votes) { +void relay_votes(void *obj, const std::vector &votes) { auto &snw = *reinterpret_cast(obj); auto my_keys_ptr = snw.core.get_service_node_keys(); @@ -217,67 +480,34 @@ static void relay_votes(void *obj, const std::vector need_remotes; - std::vector *>> valid_votes; - valid_votes.reserve(votes.size()); - for (auto &v : votes) { - auto quorum = snw.sn_list.get_quorum(v.type, v.block_height); + int votes_relayed = 0; + MDEBUG("Starting relay of " << votes.size() << " votes"); + for (auto &vote : votes) { + auto quorum = snw.core.get_service_node_list().get_quorum(vote.type, vote.block_height); if (!quorum) { - MWARNING("Unable to relay vote: no testing quorum vote for type " << v.type << " @ height " << v.block_height); + MWARNING("Unable to relay vote: no testing quorum vote for type " << vote.type << " @ height " << vote.block_height); continue; } auto &quorum_voters = quorum->validators; - if (quorum_voters.size() < service_nodes::min_votes_for_quorum_type(v.type)) { - MWARNING("Invalid vote relay: " << v.type << " quorum @ height " << v.block_height << + if (quorum_voters.size() < service_nodes::min_votes_for_quorum_type(vote.type)) { + MWARNING("Invalid vote relay: " << vote.type << " quorum @ height " << vote.block_height << " does not have enough validators (" << quorum_voters.size() << ") to reach the minimum required votes (" - << service_nodes::min_votes_for_quorum_type(v.type) << ")"); - } - - int my_pos = service_nodes::find_index_in_quorum_group(quorum_voters, my_keys.pub); - if (my_pos < 0) { - MWARNING("Invalid vote relay: vote to relay does not include this service node"); - MTRACE("me: " << my_keys.pub); - for (const auto &v : quorum->validators) - MTRACE("validator: " << v); - for (const auto &v : quorum->workers) - MTRACE("worker: " << v); + << service_nodes::min_votes_for_quorum_type(vote.type) << ")"); continue; } - for (int i : quorum_outgoing_conns(my_pos, quorum_voters.size())) - need_remotes.insert(quorum_voters[i]); - - valid_votes.emplace_back(my_pos, &v, &quorum_voters); - } - - MDEBUG("Relaying " << valid_votes.size() << " votes"); - if (valid_votes.empty()) - return; - - // pubkey => {x25519_pubkey, connect_string} - auto remotes = get_zmq_remotes(snw, need_remotes.begin(), need_remotes.end()); - - for (auto &vote_data : valid_votes) { - int my_pos = std::get<0>(vote_data); - auto &vote = *std::get<1>(vote_data); - auto &quorum_voters = *std::get<2>(vote_data); - - bt_dict vote_to_send = serialize_vote(vote); - - for (int i : quorum_outgoing_conns(my_pos, quorum_voters.size())) { - auto it = remotes.find(quorum_voters[i]); - if (it == remotes.end()) { - MINFO("Unable to relay vote to peer " << quorum_voters[i] << ": peer is inactive or we are missing a x25519 pubkey and/or quorumnet port"); - continue; - } - - auto &remote_info = it->second; - std::string x25519_pubkey{reinterpret_cast(remote_info.first.data), sizeof(crypto::x25519_public_key)}; - MDEBUG("Relaying vote to peer " << remote_info.first << " @ " << remote_info.second); - snw.snn.send(x25519_pubkey, "vote", vote_to_send, send_option::hint{remote_info.second}); + peer_info pinfo{snw, vote.type, quorum}; + if (!pinfo.my_position_count) { + MWARNING("Invalid vote relay: vote to relay does not include this service node"); + continue; } + + pinfo.relay_to_peers("vote", serialize_vote(vote)); + votes_relayed++; } + MDEBUG("Relayed " << votes_relayed << " votes"); } void handle_vote(SNNetwork::message &m, void *self) { @@ -314,10 +544,10 @@ void handle_vote(SNNetwork::message &m, void *self) { } } -/// Gets an integer value out of a bt_dict, if present and fits (i.e. get_int succeeds); if not +/// Gets an integer value out of a bt_dict, if present and fits (i.e. get_int<> succeeds); if not /// present or conversion falls, returns `fallback`. template -static std::enable_if_t::value, I> get_or(bt_dict &d, const std::string &key, I fallback) { +std::enable_if_t::value, I> get_or(bt_dict &d, const std::string &key, I fallback) { auto it = d.find(key); if (it != d.end()) { try { return get_int(it->second); } @@ -326,6 +556,210 @@ static std::enable_if_t::value, I> get_or(bt_dict &d, const return fallback; } +// Obtains the blink quorums, verifies that they are of an acceptable size, and verifies the given +// input quorum checksum matches the computed checksum for the quorums (if provided), otherwise sets +// the given output checksum (if provided) to the calculated value. Throws std::runtime_error on +// failure. +quorum_array get_blink_quorums(uint64_t blink_height, const service_node_list &snl, const uint64_t *input_checksum, uint64_t *output_checksum = nullptr) { + // We currently just use two quorums, Q and Q' in the whitepaper, but this code is designed to + // work fine with more quorums (but don't use a single subquorum; that could only be secure or + // reliable but not both). + quorum_array result; + + uint64_t local_checksum = 0; + for (uint8_t qi = 0; qi < NUM_BLINK_QUORUMS; qi++) { + auto height = blink_tx::quorum_height(blink_height, static_cast(qi)); + if (!height) + throw std::runtime_error("too early in blockchain to create a quorum"); + result[qi] = snl.get_quorum(quorum_type::blink, height); + auto &v = result[qi]->validators; + if (v.size() < BLINK_MIN_VOTES || v.size() > BLINK_SUBQUORUM_SIZE) + throw std::runtime_error("not enough blink nodes to form a quorum"); + local_checksum += quorum_checksum(v, qi * BLINK_SUBQUORUM_SIZE); + } + MTRACE("Verified enough active blink nodes for a quorum; quorum checksum: " << local_checksum); + + if (input_checksum) { + if (*input_checksum != local_checksum) + throw std::runtime_error("wrong quorum checksum: expected " + std::to_string(local_checksum) + ", received " + std::to_string(*input_checksum)); + + MTRACE("Blink quorum checksum matched"); + } + if (output_checksum) + *output_checksum = local_checksum; + + return result; +} + +// Used when debugging is enabled to print known signatures. +// Prints [x x x ...] [x x x ...] for the quorums where each "x" is either "A" for an approval +// signature, "R" for a rejection signature, or "-" for no signature. +std::string debug_known_signatures(blink_tx &btx, quorum_array &blink_quorums) { + std::ostringstream os; + bool first = true; + for (uint8_t qi = 0; qi < blink_quorums.size(); qi++) { + if (qi > 0) os << ' '; + os << '['; + const auto q = static_cast(qi); + const int slots = blink_quorums[qi]->validators.size(); + for (int i = 0; i < slots; i++) { + if (i > 0) os << ' '; + auto st = btx.get_signature_status(q, i); + os << (st == blink_tx::signature_status::approved ? 'A' : st == blink_tx::signature_status::rejected ? 'R' : '-'); + } + os << ']'; + } + return os.str(); +} + + +/// Processes blink signatures; called immediately upon receiving a signature if we know about the +/// tx; otherwise signatures are stored until we learn about the tx and then processed. +void process_blink_signatures(SNNWrapper &snw, blink_tx &btx, quorum_array &blink_quorums, uint64_t quorum_checksum, std::list &&signatures, + uint64_t reply_tag, // > 0 if we are expected to send a status update if it becomes accepted/rejected + const std::string reply_pubkey, // who we are supposed to send the status update to + const std::string &received_from = ""s /* x25519 of the peer that sent this, if available (to avoid trying to pointlessly relay back to them) */) { + + bool already_approved = false, already_rejected = false; + // First check values and discard any signatures for positions we already have. + { + auto lock = btx.shared_lock(); // Don't take out a heavier unique lock until later when we are sure we need + for (auto it = signatures.begin(); it != signatures.end(); ) { + auto &pending = *it; + auto &qi = std::get(pending); + auto &position = std::get(pending); + + auto subquorum = static_cast(qi); + auto &validators = blink_quorums[qi]->validators; + + if (position < 0 || position >= (int) validators.size()) { + MWARNING("Invalid blink signature: subquorum position is invalid"); + it = signatures.erase(it); + continue; + } + + if (btx.get_signature_status(subquorum, position) != blink_tx::signature_status::none) { + it = signatures.erase(it); + continue; + } + ++it; + } + + already_approved = btx.approved(); + already_rejected = btx.rejected(); + } + if (signatures.empty()) + return; + + // Now check and discard any invalid signatures (we can do this without holding a lock) + for (auto it = signatures.begin(); it != signatures.end(); ) { + auto &pending = *it; + auto &approval = std::get(pending); + auto &qi = std::get(pending); + auto &position = std::get(pending); + auto &signature = std::get(pending); + + auto subquorum = static_cast(qi); + auto &validators = blink_quorums[qi]->validators; + + if (!crypto::check_signature(btx.hash(approval), validators[position], signature)) { + MWARNING("Invalid blink signature: signature verification failed"); + it = signatures.erase(it); + continue; + } + ++it; + } + + if (signatures.empty()) + return; + + bool now_approved = already_approved, now_rejected = already_rejected; + { + auto lock = btx.unique_lock(); + + MTRACE("Before recording new signatures I have existing signatures: " << debug_known_signatures(btx, blink_quorums)); + + // Now actually add them (and do one last check on them) + for (auto it = signatures.begin(); it != signatures.end(); ) { + auto &pending = *it; + auto &approval = std::get(pending); + auto &qi = std::get(pending); + auto &position = std::get(pending); + auto &signature = std::get(pending); + + auto subquorum = static_cast(qi); + auto &validators = blink_quorums[qi]->validators; + + if (btx.add_prechecked_signature(subquorum, position, approval, signature)) { + MDEBUG("Validated and stored " << (approval ? "approval" : "rejection") << " signature for tx " << btx.tx.hash << ", subquorum " << int{qi} << ", position " << position); + ++it; + } + else { + // Signature already present, which means it got added between the check above and now + // by another thread. + it = signatures.erase(it); + } + } + + if (!signatures.empty()) { + now_approved = btx.approved(); + now_rejected = btx.rejected(); + MDEBUG("Updated signatures; now have signatures: " << debug_known_signatures(btx, blink_quorums)); + } + } + + if (signatures.empty()) + return; + + peer_info::exclude_set relay_exclude; + if (!received_from.empty()) { + auto pubkey = snw.core.get_service_node_list().get_pubkey_from_x25519(x25519_from_string(received_from)); + if (pubkey) + relay_exclude.insert(std::move(pubkey)); + } + + // We added new signatures that we didn't have before, so relay those signatures to blink peers + peer_info pinfo{snw, quorum_type::blink, blink_quorums.begin(), blink_quorums.end(), true /*opportunistic*/, + std::move(relay_exclude)}; + + MDEBUG("Relaying " << signatures.size() << " blink signatures to " << pinfo.strong_peers << " (strong) + " << + (pinfo.peers.size() - pinfo.strong_peers) << " (opportunistic) blink peers"); + + bt_list i_list, p_list, r_list, s_list; + for (auto &s : signatures) { + i_list.emplace_back(std::get(s)); + p_list.emplace_back(std::get(s)); + r_list.emplace_back(std::get(s)); + s_list.emplace_back(get_data_as_string(std::get(s))); + } + + bt_dict blink_sign_data{ + {"h", btx.height}, + {"#", get_data_as_string(btx.tx.hash)}, + {"q", quorum_checksum}, + {"i", std::move(i_list)}, + {"p", std::move(p_list)}, + {"r", std::move(r_list)}, + {"s", std::move(s_list)}, + }; + + pinfo.relay_to_peers("blink_sign", blink_sign_data); + + MTRACE("Done blink signature relay"); + + if (reply_tag && !reply_pubkey.empty()) { + if (now_approved && !already_approved) { + MINFO("Blink tx is now approved; sending result back to originating node"); + snw.snn.send(reply_pubkey, "bl_good", bt_dict{{"!", reply_tag}}, send_option::optional{}); + } else if (now_rejected && !already_rejected) { + MINFO("Blink tx is now rejected; sending result back to originating node"); + snw.snn.send(reply_pubkey, "bl_bad", bt_dict{{"!", reply_tag}}, send_option::optional{}); + } + } +} + + + /// A "blink" message is used to submit a blink tx from a node to members of the blink quorum and /// also used to relay the blink tx between quorum members. Fields are: /// @@ -343,9 +777,8 @@ static std::enable_if_t::value, I> get_or(bt_dict &d, const /// /// "t" - the serialized transaction data. /// -/// "#" - optional precomputed tx hash. This is included in SN-to-SN relays to allow faster -/// ignoring of already-seen transactions. It is an error if this is included and does -/// not match the actual hash of the transaction. +/// "#" - precomputed tx hash. This much match the actual hash of the transaction (the blink +/// submission will fail immediately if it does not). /// void handle_blink(SNNetwork::message &m, void *self) { auto &snw = *reinterpret_cast(self); @@ -359,12 +792,13 @@ void handle_blink(SNNetwork::message &m, void *self) { // message and close it. // If an outgoing connection - refuse reconnections via ZAP and just close it. - bool from_sn = m.from_sn(); - MDEBUG("Received a blink tx from " << (from_sn ? as_hex(m.pubkey) : "an anonymous node")); + MDEBUG("Received a blink tx from " << (m.sn ? "SN " : "non-SN ") << as_hex(m.pubkey)); + + assert(snw.core.get_service_node_keys()); if (m.data.size() != 1) { MINFO("Rejecting blink message: expected one data entry not " << m.data.size()); - // FIXME: send error response + // No valid data and so no reply tag; we can't send a response return; } auto &data = boost::get(m.data[0]); @@ -373,143 +807,143 @@ void handle_blink(SNNetwork::message &m, void *self) { // verify that height is within-2 of current height auto blink_height = get_int(data.at("h")); - auto local_height = snw.core.get_current_blockchain_height(); if (blink_height < local_height - 2) { MINFO("Rejecting blink tx because blink auth height is too low (" << blink_height << " vs. " << local_height << ")"); - // FIXME: send error response + if (tag) + m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Invalid blink authorization height"}}); return; } else if (blink_height > local_height + 2) { // TODO: if within some threshold (maybe 5-10?) we could hold it and process it once we are // within 2. MINFO("Rejecting blink tx because blink auth height is too high (" << blink_height << " vs. " << local_height << ")"); - // FIXME: send error response - return; - } - - uint64_t q_base_height = cryptonote::blink_tx::quorum_height(blink_height, cryptonote::blink_tx::subquorum::base); - if (q_base_height == 0) { - MINFO("Rejecting blink tx: too early in chain to construct a blink quorum"); - // FIXME: send error response + if (tag) + m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Invalid blink authorization height"}}); return; } + MTRACE("Blink tx auth height " << blink_height << " is valid (local height is " << local_height << ")"); auto t_it = data.find("t"); if (t_it == data.end()) { MINFO("Rejecting blink tx: no tx data included in request"); - // FIXME: send error response + if (tag) + m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "No transaction included in blink request"}}); return; } const std::string &tx_data = boost::get(t_it->second); + MTRACE("Blink tx data is " << tx_data.size() << " bytes"); // "hash" is optional -- it lets us short-circuit processing the tx if we've already seen it, // and is added internally by SN-to-SN forwards but not the original submitter. We don't trust // the hash if we haven't seen it before -- this is only used to skip propagation and // validation. - boost::optional hint_tx_hash; - if (data.count("hash")) { - auto &tx_hash_str = boost::get(data.at("hash")); - if (tx_hash_str.size() == sizeof(crypto::hash)) { - crypto::hash tx_hash; - std::memcpy(tx_hash.data, tx_hash_str.data(), sizeof(crypto::hash)); - std::lock_guard lock{snw.blinks_mutex}; - auto &umap = snw.blinks[blink_height]; + crypto::hash tx_hash; + auto &tx_hash_str = boost::get(data.at("#")); + if (tx_hash_str.size() == sizeof(crypto::hash)) { + std::memcpy(tx_hash.data, tx_hash_str.data(), sizeof(crypto::hash)); + std::shared_lock lock{snw.mutex}; + auto bit = snw.blinks.find(blink_height); + if (bit != snw.blinks.end()) { + auto &umap = bit->second; auto it = umap.find(tx_hash); - if (it != umap.end()) { + if (it != umap.end() && it->second.btxptr) { MDEBUG("Already seen and forwarded this blink tx, ignoring it."); + if (tag && !it->second.reply_tag) { + // Set the tag/pubkey if not set: that means we received it from a blink quorum + // peer before we got it from the originating node, but this is the originating + // node to whom we still want to reply. + it->second.reply_tag = tag; + it->second.reply_pubkey = m.pubkey; + } + return; } - hint_tx_hash = std::move(tx_hash); - } else { - MINFO("Rejecting blink tx: invalid tx hash included in request"); - // FIXME: send error response - return; } + MTRACE("Blink tx hash: " << as_hex(tx_hash.data)); + } else { + MINFO("Rejecting blink tx: invalid tx hash included in request"); + if (tag) + m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Invalid transaction hash"}}); + return; } - auto btxptr = std::make_shared(blink_height); + auto btxptr = std::make_shared(blink_height); auto &btx = *btxptr; - // We currently just use two quorums, Q and Q' in the whitepaper, but this code is designed to - // work fine with more quorums (but don't use a single subquorum; that could only be secure or - // reliable but not both). - constexpr auto NUM_QUORUMS = tools::enum_count; - std::array, NUM_QUORUMS> blink_quorums = {}; - - uint64_t local_checksum = 0; - for (uint8_t qi = 0; qi < NUM_QUORUMS; qi++) { - auto height = btx.quorum_height(static_cast(qi)); - blink_quorums[qi] = snw.sn_list.get_quorum(quorum_type::blink, height); - - local_checksum += quorum_checksum(blink_quorums[qi]->validators, qi * BLINK_SUBQUORUM_SIZE); - } - - if (!std::all_of(blink_quorums.begin(), blink_quorums.end(), - [](const auto &quo) { auto v = quo->validators.size(); return v >= BLINK_MIN_VOTES && v <= BLINK_SUBQUORUM_SIZE; })) { - MINFO("Rejecting blink tx: not enough blink nodes to form a quorum"); - // FIXME: send error response + quorum_array blink_quorums; + uint64_t checksum = get_int(data.at("q")); + try { + blink_quorums = get_blink_quorums(blink_height, snw.core.get_service_node_list(), &checksum); + } catch (const std::runtime_error &e) { + MINFO("Rejecting blink tx: " << e.what()); + if (tag) + m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Unable to retrieve blink quorum: "s + e.what()}}); return; } - auto input_checksum = get_int(data.at("q")); - if (input_checksum != local_checksum) { - MINFO("Rejecting blink tx: wrong quorum checksum"); - // FIXME: send error response + peer_info pinfo{snw, quorum_type::blink, blink_quorums.begin(), blink_quorums.end(), true /*opportunistic*/, + {snw.core.get_service_node_list().get_pubkey_from_x25519(x25519_from_string(m.pubkey))} // exclude the peer that just sent it to us + }; + + if (pinfo.my_position_count > 0) + MTRACE("Found this SN in " << pinfo.my_position_count << " subquorums"); + else { + MINFO("Rejecting blink tx: this service node is not a member of the blink quorum!"); + if (tag) + m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Blink tx relayed to non-blink quorum member"}}); return; } - auto keys = snw.core.get_service_node_keys(); - const auto &my_pubkey = keys->pub; + { + crypto::hash tx_hash_actual; + if (!cryptonote::parse_and_validate_tx_from_blob(tx_data, btx.tx, tx_hash_actual)) { + MINFO("Rejecting blink tx: failed to parse transaction data"); + if (tag) + m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Failed to parse transaction data"}}); + return; + } + MTRACE("Successfully parsed transaction data"); - std::array mypos; - mypos.fill(-1); - - for (size_t qi = 0; qi < blink_quorums.size(); qi++) { - auto &v = blink_quorums[qi]->validators; - for (size_t pki = 0; pki < v.size(); pki++) { - if (v[pki] == my_pubkey) { - mypos[qi] = pki; - break; - } + if (tx_hash != tx_hash_actual) { + MINFO("Rejecting blink tx: submitted tx hash " << tx_hash << " did not match actual tx hash " << tx_hash_actual); + if (tag) + m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "Invalid transaction hash"}}); + return; + } else { + MTRACE("Pre-computed tx hash matches actual tx hash"); } } - if (std::none_of(mypos.begin(), mypos.end(), [](auto pos) { return pos >= 0; })) { - MINFO("Rejecting blink tx: this service node is not a member of the blink quorum!"); - // FIXME: send error response + // Abort if we don't have at least one strong peer to send it to. This can only happen if it's + // a brand new SN (not just restarted!) that hasn't received uptime proofs before. + if (!pinfo.strong_peers) { + MWARNING("Could not find connection info for any blink quorum peers. Aborting blink tx"); + if (tag) + m.reply("bl_nostart", bt_dict{{"!", tag}, {"e", "No quorum peers are currently reachable"}}); return; } - crypto::hash tx_hash; - if (!cryptonote::parse_and_validate_tx_from_blob(tx_data, btx.tx(), tx_hash)) { - MINFO("Rejecting blink tx: failed to parse transaction data"); - // FIXME: send error response - return; - } - - if (hint_tx_hash && tx_hash != *hint_tx_hash) { - MINFO("Rejecting blink tx: hint tx hash did not match actual tx hash"); - // FIXME: send error response - return; - } - - // See if we've already handled this blink tx. We do this even if we already checked the hint - // hash to avoid a race condition between there are here that could result in two nearly - // simultaneous blink causing the blink to be forwarded multiple times. + // See if we've already handled this blink tx, and if not, store it. Also check for any pending + // signatures for this blink tx that we received or processed before we got here with this tx. + std::list signatures; { - std::lock_guard lock(snw.blinks_mutex); - auto &umap = snw.blinks[blink_height]; - auto it = umap.find(tx_hash); - if (it != umap.end()) { + std::unique_lock lock(snw.mutex); + auto &bl_info = snw.blinks[blink_height][tx_hash]; + if (bl_info.btxptr) { MDEBUG("Already seen and forwarded this blink tx, ignoring it."); return; } - + bl_info.btxptr = btxptr; + for (auto &sig : bl_info.pending_sigs) + signatures.push_back(std::move(sig)); + bl_info.pending_sigs.clear(); + if (tag > 0) { + bl_info.reply_tag = tag; + bl_info.reply_pubkey = m.pubkey; + } } - - - // TODO: Reply here to say we've accepted it for verification + MTRACE("Accepted new blink tx for verification"); // The submission looks good. We distribute it first, *before* we start verifying the actual tx // details, for two reasons: we want other quorum members to start verifying ASAP, and we want @@ -520,101 +954,405 @@ void handle_blink(SNNetwork::message &m, void *self) { // FIXME - am I 100% sure I want to do the above? Verifying the TX would cut off being able to // induce a node to broadcast a junk TX to other quorum members. - std::unordered_set need_remotes; - for (auto &q : blink_quorums) - for (auto &pubkey : q->validators) - need_remotes.insert(pubkey); - // pubkey => {x25519_pubkey, connect_string} - auto remotes = get_zmq_remotes(snw, need_remotes.begin(), need_remotes.end()); - - bt_dict relay_data{ - {"h", blink_height}, - {"q", local_checksum}, - {"tx", tx_data}, - {"hash", std::string{tx_hash.data, sizeof(tx_hash.data)}}, - }; - auto relay_blink = [&remotes, &snw, &relay_data](const auto &pubkey, bool optional = false) { - auto it = remotes.find(pubkey); - if (it == remotes.end()) { - MINFO("Unable to relay blink tx to service node " << pubkey << ": service node is inactive or has not sent a x25519 pubkey, ip, and/or quorumnet port"); - return; - } - - auto &remote_info = it->second; - std::string x25519_pubkey{reinterpret_cast(remote_info.first.data), sizeof(crypto::x25519_public_key)}; - MDEBUG("Relaying blink tx to peer " << remote_info.first << " @ " << remote_info.second); - if (optional) - snw.snn.send(x25519_pubkey, "blink", relay_data, send_option::optional{}); - else - snw.snn.send(x25519_pubkey, "blink", relay_data, send_option::hint{remote_info.second}); - }; - - for (size_t i = 0; i < mypos.size(); i++) { - if (mypos[i] < 0) - continue; - - // TODO: when we receive a new block, if our quorum starts soon we can tell SNNetwork to - // pre-connect (to save the time in handshaking when we get an actual blink tx). - - auto &quorum = *blink_quorums[i]; - - // Relay to all my outgoing targets within the quorum (connecting if not already connected) - for (int j : quorum_outgoing_conns(mypos[i], quorum.validators.size())) { - MTRACE("Intra-quorum blink relay to " << quorum.validators[j]); - relay_blink(quorum.validators[j]); - } - - // Relay to all my *incoming* sources within the quorum *if* I already have a connection - // open with them, but don't open a new connection if I don't. - for (int j : quorum_incoming_conns(mypos[i], quorum.validators.size())) { - MTRACE("Intra-quorum optional blink relay to " << quorum.validators[j]); - relay_blink(quorum.validators[j], true /*optional*/); - } - - // If I'm in the last half* of the first quorum then I relay to the first half (roughly) of - // the next quorum. i.e. nodes 5-9 in Q send to nodes 0-4 in Q'. For odd numbers the last - // position gets left out (e.g. for 9 members total we would have 0-3 talk to 4-7 and no one - // talks to 8). - // - // (* - half here means half the size of the smaller quorum) - // - // We also skip this entirely if this SN is in both quorums since then we're already - // relaying to nodes in the next quorum. (Ideally we'd do the same if the recipient is in - // both quorums, but that's harder to figure out and so the special case isn't worth - // worrying about). - if (i + 1 < mypos.size() && mypos[i + 1] < 0) { - auto &next_quorum = *blink_quorums[i + 1]; - int half = std::min(quorum.validators.size(), next_quorum.validators.size()) / 2; - if (mypos[i] >= half && mypos[i] < half*2) { - MTRACE("Inter-quorum relay from Q to Q' service node " << next_quorum.validators[mypos[i] - half]); - relay_blink(next_quorum.validators[mypos[i] - half]); - } - } - - // Exactly the same connections as above, but in reverse: the first half of Q' sends to the - // second half of Q. Typically this will end up reusing an already open connection, but if - // there isn't such an open connection then we establish a new one. (We could end up with - // one each way, but that won't hurt anything). - if (i > 0 && mypos[i - 1] < 0) { - auto &prev_quorum = *blink_quorums[i - 1]; - int half = std::min(quorum.validators.size(), prev_quorum.validators.size()) / 2; - if (mypos[i] < half) { - MTRACE("Inter-quorum relay from Q' to Q service node " << prev_quorum.validators[mypos[i] + half]); - relay_blink(prev_quorum.validators[mypos[i] + half]); - } - } - - // Note: don't break here: it's possible for us to land in both quorums, which is fine (and likely on testnet) + { + bt_dict blink_data{ + {"h", blink_height}, + {"q", checksum}, + {"t", tx_data}, + {"#", tx_hash_str}, + }; + MDEBUG("Relaying blink tx to " << pinfo.strong_peers << " strong and " << (pinfo.peers.size() - pinfo.strong_peers) << " opportunistic blink peers"); + pinfo.relay_to_peers("blink", blink_data); } - // Lock key images. + // Anything past this point always results in a success or failure signature getting sent to peers + + // Check tx for validity + bool already_in_mempool; + cryptonote::tx_verification_context tvc = {}; + bool approved = snw.pool.add_blink(btxptr, tvc, already_in_mempool); + + MINFO("Blink TX " << tx_hash << (approved ? " approved and added to mempool" : " rejected")); + if (!approved) + MDEBUG("TX rejected because: " << print_tx_verification_context(tvc)); + + auto hash_to_sign = btx.hash(approved); + auto &keys = *snw.core.get_service_node_keys(); + crypto::signature sig; + generate_signature(hash_to_sign, keys.pub, keys.key, sig); + + // Now that we have the blink tx stored we can add our signature *and* any other pending + // signatures we are holding onto, then blast the entire thing to our peers. + for (uint8_t qi = 0; qi < NUM_BLINK_QUORUMS; qi++) { + if (pinfo.my_position[qi] < 0) + continue; + signatures.emplace_back(approved, qi, pinfo.my_position[qi], sig); + } + + process_blink_signatures(snw, btx, blink_quorums, checksum, std::move(signatures), tag, m.pubkey); } -} // empty namespace +template +void copy_signature_values(std::list &signatures, const bt_value &val, CopyValue copy_value) { + auto &results = boost::get(val); + if (signatures.empty()) + signatures.resize(results.size()); + else if (results.empty()) + throw std::invalid_argument("Invalid blink signature data: no signatures sent"); + else if (signatures.size() != results.size()) + throw std::invalid_argument("Invalid blink signature data: i, p, r, s lengths must be identical"); + auto it = signatures.begin(); + for (auto &r : results) + copy_value(std::get(*it++), r); +} + +/// A "blink_sign" message is used to relay signatures from one quorum member to other members. +/// Fields are: +/// +/// "h" - Blink authorization height of the signature. +/// +/// "#" - tx hash of the transaction. +/// +/// "q" - checksum of blink quorum members. Mandatory, and must match the receiving SN's +/// locally computed checksum of blink quorum members. +/// +/// "i" - list of quorum indices, i.e. 0 for the base quorum, 1 for the future quorum +/// +/// "p" - list of quorum positions +/// +/// "r" - list of blink signature results (0 if rejected, 1 if approved) +/// +/// "s" - list of blink signatures +/// +/// Each of "i", "p", "r", and "s" must be exactly the same length; each element at a position in +/// each list corresponds to the values at the same position of the other lists. +/// +/// Signatures will be forwarded if new; known signatures will be ignored. +void handle_blink_signature(SNNetwork::message &m, void *self) { + auto &snw = *reinterpret_cast(self); + + MDEBUG("Received a blink tx signature from SN " << as_hex(m.pubkey)); + + if (m.data.size() != 1) + throw std::runtime_error("Rejecting blink signature: expected one data entry not " + std::to_string(m.data.size())); + + auto &data = boost::get(m.data[0]); + + uint64_t blink_height = 0, checksum = 0; + crypto::hash tx_hash; + bool saw_checksum = false, saw_hash = false, saw_i, saw_r, saw_p, saw_s; + std::list signatures; + + for (const auto &input : data) { + if (input.first.size() != 1) + throw std::invalid_argument("Invalid blink signature data: invalid/unrecognized key " + input.first); + + auto &val = input.second; + switch (input.first[0]) { + case 'h': + blink_height = get_int(val); + break; + case '#': { + auto &hash_str = boost::get(val); + if (hash_str.size() != sizeof(crypto::hash)) + throw std::invalid_argument("Invalid blink signature data: invalid tx hash"); + std::memcpy(tx_hash.data, hash_str.data(), sizeof(crypto::hash)); + saw_hash = true; + break; + } + case 'q': + checksum = get_int(val); + saw_checksum = true; + break; + case 'i': + copy_signature_values(signatures, val, [](uint8_t &dest, const bt_value &v) { + dest = get_int(v); + if (dest >= NUM_BLINK_QUORUMS) + throw std::invalid_argument("Invalid blink signature data: invalid quorum index " + std::to_string(dest)); + }); + saw_i = true; + break; + case 'r': + copy_signature_values(signatures, val, [](bool &dest, const bt_value &v) { dest = get_int(v); }); + saw_r = true; + break; + case 'p': + copy_signature_values(signatures, val, [](int &dest, const bt_value &v) { + dest = get_int(v); + if (dest < 0 || dest >= BLINK_SUBQUORUM_SIZE) // This is only input validation: it might actually have to be smaller depending on the actual quorum (we check later) + throw std::invalid_argument("Invalid blink signature data: invalid quorum position " + std::to_string(dest)); + }); + saw_p = true; + break; + case 's': + copy_signature_values(signatures, val, [](crypto::signature &dest, const bt_value &v) { + auto &sig_str = boost::get(v); + if (sig_str.size() != sizeof(crypto::signature)) + throw std::invalid_argument("Invalid blink signature data: invalid signature"); + std::memcpy(&dest, sig_str.data(), sizeof(crypto::signature)); + if (!dest) + throw std::invalid_argument("Invalid blink signature data: invalid null signature"); + }); + saw_s = true; + break; + default: + throw std::invalid_argument("Invalid blink signature data: invalid/unrecognized key " + input.first); + } + } + + if (!(blink_height && saw_hash && saw_checksum && saw_i && saw_r && saw_p && saw_s)) + throw std::invalid_argument("Invalid blink signature data: missing required fields"); + + auto blink_quorums = get_blink_quorums(blink_height, snw.core.get_service_node_list(), &checksum); // throws if bad quorum or checksum mismatch + + uint64_t reply_tag; + std::string reply_pubkey; + std::shared_ptr btxptr; + { + std::shared_lock lock{snw.mutex}; + auto bit = snw.blinks.find(blink_height); + if (bit != snw.blinks.end()) { + auto it = bit->second.find(tx_hash); + if (it != bit->second.end()) { + btxptr = it->second.btxptr; + reply_tag = it->second.reply_tag; + reply_pubkey = it->second.reply_pubkey; + } + } + } + + if (btxptr) + MINFO("Found blink tx in local blink cache"); + else { + MINFO("Blink tx not found in local blink cache; delaying signature verification"); + std::unique_lock lock{snw.mutex}; + auto &delayed = snw.blinks[blink_height][tx_hash].pending_sigs; + for (auto &sig : signatures) + delayed.insert(std::move(sig)); + return; + } + + process_blink_signatures(snw, *btxptr, blink_quorums, checksum, std::move(signatures), reply_tag, reply_pubkey, m.pubkey); +} -namespace quorumnet { +// tag -> {hash, promise, expiry} +using blink_response = std::pair; +struct blink_result_data { + crypto::hash hash; + std::promise promise; + std::chrono::high_resolution_clock::time_point expiry; + int remote_count; + std::atomic nostart_count{0}; + std::atomic bad_count{0}; + std::atomic good_count{0}; +}; +std::unordered_map pending_blink_results; +std::shared_timed_mutex pending_blink_result_mutex; + +// Sanity check against runaway active pending blink submissions +constexpr size_t MAX_ACTIVE_PROMISES = 1000; + +std::future> send_blink(void *obj, const std::string &tx_blob) { + std::promise> promise; + auto future = promise.get_future(); + cryptonote::transaction tx; + crypto::hash tx_hash; + + thread_local std::mt19937_64 rng{std::random_device{}()}; + uint64_t blink_tag = 0; + blink_result_data *brd = nullptr; + + if (!cryptonote::parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash)) { + promise.set_value(std::make_pair(cryptonote::blink_result::rejected, "Could not parse transaction data")); + } else { + auto now = std::chrono::high_resolution_clock::now(); + bool found = false; + std::unique_lock lock{pending_blink_result_mutex}; + for (auto it = pending_blink_results.begin(); it != pending_blink_results.end(); ) { + auto &brd = it->second; + if (brd.expiry >= now) { + try { brd.promise.set_value(std::make_pair(cryptonote::blink_result::timeout, "Blink quorum timeout")); } + catch (const std::future_error &) { /* ignore */ } + it = pending_blink_results.erase(it); + } else { + if (!found && brd.hash == tx_hash) + found = true; + ++it; + } + } + if (found) { + promise.set_value(std::make_pair(cryptonote::blink_result::rejected, "Transaction was already submitted")); + } else if (pending_blink_results.size() >= MAX_ACTIVE_PROMISES) { + promise.set_value(std::make_pair(cryptonote::blink_result::rejected, "Node is busy, try again later")); + } else { + while (!brd) { + // Choose an unused tag randomly so that the blink tag value doesn't give anything away + blink_tag = rng(); + if (blink_tag == 0 || pending_blink_results.count(blink_tag) > 0) continue; + brd = &pending_blink_results[blink_tag]; + brd->hash = tx_hash; + brd->promise = std::move(promise); + brd->expiry = std::chrono::high_resolution_clock::now() + 30s; + } + } + } + + if (blink_tag > 0) { + auto &snw = *reinterpret_cast(obj); + uint64_t height = snw.core.get_current_blockchain_height(); + uint64_t checksum; + auto quorums = get_blink_quorums(height, snw.core.get_service_node_list(), nullptr, &checksum); + + // Lookup the x25519 and ZMQ connection string for all possible blink recipients so that we + // know where to send it to, and so that we can immediately exclude SNs that aren't active + // anymore. + std::unordered_set candidates; + for (auto &q : quorums) + candidates.insert(q->validators.begin(), q->validators.end()); + + MDEBUG("Have " << candidates.size() << " blink SN candidates"); + + std::vector> remotes; // x25519 pubkey -> connect string + remotes.reserve(candidates.size()); + snw.core.get_service_node_list().for_each_service_node_info(candidates.begin(), candidates.end(), + [&remotes](const crypto::public_key &pubkey, const service_nodes::service_node_info &info) { + if (!info.is_active()) { + MTRACE("Not include inactive node " << pubkey); + return; + } + auto &proof = *info.proof; + if (!proof.pubkey_x25519 || !proof.quorumnet_port || !proof.public_ip) { + MTRACE("Not including node " << pubkey << ": missing x25519(" << as_hex(get_data_as_string(proof.pubkey_x25519)) << "), " + "public_ip(" << epee::string_tools::get_ip_string_from_int32(proof.public_ip) << "), or qnet port(" << proof.quorumnet_port << ")"); + return; + } + remotes.emplace_back(get_data_as_string(proof.pubkey_x25519), + "tcp://" + epee::string_tools::get_ip_string_from_int32(proof.public_ip) + ":" + std::to_string(proof.quorumnet_port)); + }); + + MDEBUG("Have " << remotes.size() << " blink SN candidates after checking active status and connection details"); + + // Select 4 random (active) blink quorum SNs to send the blink to. + std::vector indices(remotes.size()); + std::iota(indices.begin(), indices.end(), 0); + std::shuffle(indices.begin(), indices.end(), rng); + if (indices.size() > 4) + indices.resize(4); + brd->remote_count = indices.size(); + + send_option::serialized data{bt_dict{ + {"!", blink_tag}, + {"#", get_data_as_string(tx_hash)}, + {"h", height}, + {"q", checksum}, + {"t", tx_blob} + }}; + + for (size_t i : indices) { + MINFO("Relaying blink tx to " << as_hex(remotes[i].first) << " @ " << remotes[i].second); + snw.snn.send(remotes[i].first, "blink", data, send_option::hint{remotes[i].second}); + } + } + + return future; +} + + +void common_blink_response(uint64_t tag, cryptonote::blink_result res, std::string msg, bool nostart, bool bad, bool good) { + assert(int{nostart} + int{bad} + int{good} == 1); + bool promise_set = false; + { + std::shared_lock lock{pending_blink_result_mutex}; + auto it = pending_blink_results.find(tag); + if (it == pending_blink_results.end()) + return; // Already handled, or obsolete + + auto &pbr = it->second; + auto &count = nostart ? pbr.nostart_count : bad ? pbr.bad_count : pbr.good_count; + auto count_same = ++count; + if (count_same > pbr.remote_count / 2) { + try { + pbr.promise.set_value(std::make_pair(res, msg)); + promise_set = true; + } + catch (const std::future_error &) { /* ignore */ } + } + } + if (promise_set) { + std::unique_lock lock{pending_blink_result_mutex}; + pending_blink_results.erase(tag); + } +} + +/// bl_nostart is sent back to the submitter when the tx doesn't get far enough to be distributed +/// among the quorum because of some failure (bad height, parse failure, etc.) It includes: +/// +/// ! - the tag as included in the submission +/// e - an error message +/// +/// It's possible for some nodes to accept and others to refuse, so we don't actually set the +/// promise unless we get a nostart response from a majority of the remotes. +void handle_blink_not_started(SNNetwork::message &m, void *self) { + if (m.data.size() != 1) { + MERROR("Bad blink not started response: expected one data entry not " << m.data.size()); + return; + } + auto &data = boost::get(m.data[0]); + auto tag = get_int(data.at("!")); + auto &error = boost::get(data.at("e")); + + MINFO("Received no-start blink response: " << error); + + common_blink_response(tag, cryptonote::blink_result::rejected, std::move(error), true, false, false); +} +/// bl_bad gets returned once we know enough of the blink quorum has rejected the result to make it +/// unequivocal that it has been rejected. We require a failure response from a majority of the +/// remotes before setting the promise. +/// +/// ! - the tag as included in the submission +/// +void handle_blink_failure(SNNetwork::message &m, void *self) { + if (m.data.size() != 1) { + MERROR("Blink failure message not understood: expected one data entry not " << m.data.size()); + return; + } + auto &data = boost::get(m.data[0]); + auto tag = get_int(data.at("!")); + + // TODO - we ought to be able to signal an error message *sometimes*, e.g. if one of the remotes + // we sent it to rejected it then that remote can reply with a message. That gets a bit + // complicated, though, in terms of maintaining internal state (since the bl_bad is sent on + // signature receipt, not at rejection time), so for now we don't include it. + //auto &error = boost::get(data.at("e")); + + MINFO("Received blink failure response"); + + common_blink_response(tag, cryptonote::blink_result::rejected, "Transaction rejected by quorum"s, false, true, false); +} + +/// bl_good gets returned once we know enough of the blink quorum has accepted the result to make it +/// valid. We require a good response from a majority of the remotes before setting the promise. +/// +/// ! - the tag as included in the submission +/// +void handle_blink_success(SNNetwork::message &m, void *self) { + if (m.data.size() != 1) { + MERROR("Blink success message not understood: expected one data entry not " << m.data.size()); + return; + } + auto &data = boost::get(m.data[0]); + auto tag = get_int(data.at("!")); + + MINFO("Received blink success response"); + + common_blink_response(tag, cryptonote::blink_result::accepted, ""s, false, false, true); +} + + +} // end empty namespace + /// Sets the cryptonote::quorumnet_* function pointers (allowing core to avoid linking to /// cryptonote_protocol). Called from daemon/daemon.cpp. Also registers quorum command callbacks. @@ -622,6 +1360,7 @@ void init_core_callbacks() { cryptonote::quorumnet_new = new_snnwrapper; cryptonote::quorumnet_delete = delete_snnwrapper; cryptonote::quorumnet_relay_votes = relay_votes; + cryptonote::quorumnet_send_blink = send_blink; // Receives a vote SNNetwork::register_quorum_command("vote", handle_vote); @@ -630,29 +1369,24 @@ void init_core_callbacks() { // members who received it from an external node. SNNetwork::register_public_command("blink", handle_blink); - // Sends a message back to the blink initiator that the transaction was accepted for relaying. - // This is only sent by the entry point service nodes into the quorum to let it know the tx - // looks good enough to pass to other quorum members, but it does *not* indicate approval. -// SNNetwork::register_quorum_command("bl_start", handle_blink_started); - // Sends a message back to the blink initiator that the transaction was NOT relayed, either // because the height was invalid or the quorum checksum failed. This is only sent by the entry // point service nodes into the quorum to let it know the tx verification has not started from // that node. It does not necessarily indicate a failure unless all entry point attempts return // the same. -// SNNetwork::register_quorum_command("bl_nostart", handle_blink_not_started); + SNNetwork::register_quorum_command("bl_nostart", handle_blink_not_started); // Sends a message from the entry SNs back to the initiator that the Blink tx has been rejected: - // that is, enough signing rejections have occured that the Blink TX cannot proceed. -// SNNetwork::register_quorum_command("bl_bad", handle_blink_failure); + // that is, enough signed rejections have occured that the Blink tx cannot be accepted. + SNNetwork::register_quorum_command("bl_bad", handle_blink_failure); // Sends a message from the entry SNs back to the initiator that the Blink tx has been accepted - // and is being broadcast to the network. -// SNNetwork::register_quorum_command("bl_good", handle_blink_success); + // and validated and is being broadcast to the network. + SNNetwork::register_quorum_command("bl_good", handle_blink_success); // Receives blink tx signatures or rejections between quorum members (either original or // forwarded). These are propagated by the receiver if new -// SNNetwork::register_quorum_command("blink_sign", handle_blink_signature); + SNNetwork::register_quorum_command("blink_sign", handle_blink_signature); } } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index f6f31969a..8d2638799 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -2526,11 +2526,11 @@ static void append_printable_service_node_list_entry(cryptonote::network_type ne } stream << "\n"; - stream << indent2 << "IP Address & Port: "; + stream << indent2 << "IP Address & Ports: "; if (entry.public_ip == "0.0.0.0") stream << "(Awaiting confirmation from network)"; else - stream << entry.public_ip << ":" << entry.storage_port; + stream << entry.public_ip << " :" << entry.storage_port << " (storage), :" << entry.quorumnet_port << " (quorumnet)"; stream << "\n"; stream << indent2 << "Storage Server Reachable: " << (entry.storage_server_reachable ? "Yes" : "No") << " ("; diff --git a/src/quorumnet/sn_network.cpp b/src/quorumnet/sn_network.cpp index f41176014..58b15f06b 100644 --- a/src/quorumnet/sn_network.cpp +++ b/src/quorumnet/sn_network.cpp @@ -28,19 +28,19 @@ constexpr char ZMQ_ADDR_ZAP[] = "inproc://zeromq.zap.01"; constexpr int SN_HANDSHAKE_TIME = 10000; /** Maximum incoming message size; if a remote tries sending a message larger than this they get disconnected */ -constexpr int64_t SN_ZMQ_MAX_MSG_SIZE = 4 * 1024 * 1024; +constexpr int64_t SN_ZMQ_MAX_MSG_SIZE = 1 * 1024 * 1024; +/** How long (in ms) to linger sockets when closing them; this is the maximum time zmq spends trying + * to sending pending messages before dropping them and closing the underlying socket after the + * high-level zmq socket is closed. */ +constexpr int CLOSE_LINGER = 5000; // Inside some method: // SN_LOG(warn, "bad" << 42 << "stuff"); #define SN_LOG(level, stuff) do { if (want_logs(LogLevel::level)) { std::ostringstream o; o << stuff; logger(LogLevel::level, __FILE__, __LINE__, o.str()); } } while (0) -// This is the domain used for service nodes talking to each other, and for the service node in a -// node -> service node communication. +// This is the domain used for listening service nodes. constexpr const char AUTH_DOMAIN_SN[] = "loki.sn"; -// This is the domain for the node in a node -> service node communication, and requires no client -// authentication (the SN still authenticates to the client though). -constexpr const char AUTH_DOMAIN_CLIENT[] = "loki.node"; #ifdef __cpp_lib_string_view using msg_view_t = std::string_view; @@ -200,11 +200,12 @@ template void forward_to_worker(zmq::socket_t &workers, std::string worker_id, It parts_begin, It parts_end) { assert(parts_begin != parts_end); - // Forwarded message to worker: start with the worker name (so the worker router + // Forwarded message to worker: this just sends all the parts, but prefixed with the worker + // name. + // start with the worker name (so the worker router // knows where to send it), then the authenticated remote pubkey, then the message // parts. workers.send(create_message(std::move(worker_id)), zmq::send_flags::sndmore); -// workers.send(create_message(std::move(sender)), zmq::send_flags::sndmore); send_message_parts(workers, parts_begin, parts_end); } @@ -296,16 +297,43 @@ SNNetwork::SNNetwork( WantLog want_log, WriteLog log, unsigned int max_workers) - : object_id{next_id++}, peer_lookup{std::move(lookup)}, allow_connection{std::move(allow)}, want_logs{want_log}, logger{log}, max_workers{max_workers}, pubkey{std::move(pubkey_)}, privkey{std::move(privkey_)} + : object_id{next_id++}, pubkey{std::move(pubkey_)}, privkey{std::move(privkey_)}, peer_lookup{std::move(lookup)}, allow_connection{std::move(allow)}, + want_logs{want_log}, logger{log}, poll_remote_offset{poll_internal_size + 1}, max_workers{max_workers} { - SN_LOG(trace, "Constructing SNNetwork, id=" << object_id << ", this=" << this); + SN_LOG(trace, "Constructing listening SNNetwork, id=" << object_id << ", this=" << this); if (bind.empty()) - throw std::invalid_argument{"Cannot create a service node with no address(es) to bind"}; + throw std::invalid_argument{"Cannot create a service node listener with no address(es) to bind"}; + + listener = std::make_unique(context, zmq::socket_type::router); + + launch_proxy_thread(bind); +} + +SNNetwork::SNNetwork( + LookupFunc lookup, + AllowFunc allow, + WantLog want_log, + WriteLog log, + unsigned int max_workers) + : object_id{next_id++}, peer_lookup{std::move(lookup)}, allow_connection{std::move(allow)}, + want_logs{want_log}, logger{log}, poll_remote_offset{poll_internal_size}, max_workers{max_workers} +{ + SN_LOG(trace, "Constructing remote-only SNNetwork, id=" << object_id << ", this=" << this); + + SN_LOG(debug, "generating x25519 keypair for remote-only SNNetwork instance"); + pubkey.resize(crypto_box_PUBLICKEYBYTES); + privkey.resize(crypto_box_SECRETKEYBYTES); + crypto_box_keypair(reinterpret_cast(&pubkey[0]), reinterpret_cast(&privkey[0])); + + launch_proxy_thread({}); +} + +void SNNetwork::launch_proxy_thread(const std::vector &bind) { commands_mutable = false; - SN_LOG(info, "Initializing SNNetwork quorumnet listener with pubkey " << as_hex(pubkey)); + SN_LOG(info, "Initializing SNNetwork quorumnet " << (bind.empty() ? "remote-only" : "listener") << " with pubkey " << as_hex(pubkey)); assert(pubkey.size() == 32 && privkey.size() == 32); if (max_workers == 0) @@ -317,7 +345,7 @@ SNNetwork::SNNetwork( command.bind(SN_ADDR_COMMAND); proxy_thread = std::thread{&SNNetwork::proxy_loop, this, bind}; - SN_LOG(warn, "Waiting for proxy thread to get ready..."); + SN_LOG(debug, "Waiting for proxy thread to get ready..."); auto &control = get_control_socket(); detail::send_control(control, "START"); SN_LOG(trace, "Sent START command"); @@ -329,10 +357,11 @@ SNNetwork::SNNetwork( if (!(parts.size() == 1 && view(parts.front()) == "READY")) throw std::runtime_error("Invalid startup message from proxy thread (didn't get expected READY message)"); - SN_LOG(warn, "Proxy thread is ready"); + SN_LOG(debug, "Proxy thread is ready"); } } + void SNNetwork::spawn_worker(std::string id) { worker_threads.emplace(std::piecewise_construct, std::make_tuple(id), std::make_tuple(&SNNetwork::worker_thread, this, id)); } @@ -348,14 +377,18 @@ void SNNetwork::worker_thread(std::string worker_id) { // When we get an incoming message it'll be in parts that look like one of: // [CONTROL] -- some control command, e.g. "QUIT" - // [PUBKEY, CMD] -- some simple command with no arguments - // [PUBKEY, CMD, DATA] -- some data-carrying command where DATA is a serialized bt_dict - // ["", ROUTE, CMD], ["", ROUTE, CMD, DATA] -- same as above, but for a command originating - // from a non-SN source. + // [PUBKEY, "S", CMD, DATA...] + // some data-carrying command where DATA... is zero or more serialized bt_values, where + // the command originated from a known service node (i.e. that we can reconnect to, if + // needed, to relay a message). + // [PUBKEY, "C", CMD, DATA...] + // same as above, but for a command originating from a non-SN source (e.g. a remote node + // submitting a blink TX). We have a local route to send back to the source as long as + // it stays connecting but cannot reconnect to it. // // CMDs are registered *before* a SNNetwork is created and immutable afterwards and have - // an associated callback that takes the pubkey and a bt_dict (for simple commands we pass - // an empty bt_dict). + // an associated callback that takes the pubkey and a vector of deserialized + // DATA... values. SN_LOG(debug, "worker " << worker_id << " waiting for requests"); std::list parts; recv_message_parts(sock, std::back_inserter(parts)); @@ -375,35 +408,39 @@ void SNNetwork::worker_thread(std::string worker_id) { } } + assert(parts.size() >= 3); + auto pubkey = pop_string(parts); - std::string route; - if (pubkey.empty()) - route = pop_string(parts); + assert(pubkey.size() == 32); + bool sn = pop_string(parts) == "S"; - if (parts.size() < 1 || (pubkey.empty() ? route.empty() : pubkey.size() != 32)) { - // proxy shouldn't have let this through! - SN_LOG(error, worker_id << "received malformed message from proxy thread: " << parts.size() << " message frames, pubkey size " << pubkey.size() << ", route? " << !route.empty()); - continue; - } - - message msg{*this, pop_string(parts), std::move(pubkey), std::move(route)}; - SN_LOG(trace, worker_id << " received " << msg.command << " message from " << (msg.from_sn() ? as_hex(msg.pubkey) : "non-SN remote"s) << + message msg{*this, pop_string(parts), std::move(pubkey), sn}; + SN_LOG(trace, worker_id << " received " << msg.command << " message from " << (msg.sn ? "SN " : "non-SN ") << as_hex(msg.pubkey) << " with " << parts.size() << " data parts"); for (const auto &part : parts) { msg.data.emplace_back(); bt_deserialize(part.data(), part.size(), msg.data.back()); } + if (msg.command == "BYE") { + SN_LOG(info, "peer asked us to disconnect"); + detail::send_control(get_control_socket(), "DISCONNECT", {{"pubkey",msg.pubkey}}); + continue; + } + auto cmdit = commands.find(msg.command); if (cmdit == commands.end()) { - SN_LOG(warn, worker_id << " received unknown command '" << msg.command << "' from SN " << - (msg.from_sn() ? as_hex(msg.pubkey) : "non-SN remote"s)); + SN_LOG(warn, worker_id << " received unknown command '" << msg.command << "' from " << + (msg.sn ? "SN " : "non-SN ") << as_hex(msg.pubkey)); continue; } const bool &public_cmd = cmdit->second.second; - if (!public_cmd && !msg.from_sn()) { - SN_LOG(warn, worker_id << " (of " << object_id << ") received quorum-only command from an unauthenticated remote; ignoring"); + if (!public_cmd && !msg.sn) { + // If they aren't valid, tell them so that they can disconnect (and attempt to reconnect later with appropriate authentication) + SN_LOG(warn, worker_id << " (of " << object_id << ") received quorum-only command from an non-SN authenticated remote; replying with a BYE"); + send(msg.pubkey, "BYE", send_option::incoming{}); + detail::send_control(get_control_socket(), "DISCONNECT", {{"pubkey",msg.pubkey}}); continue; } @@ -431,8 +468,6 @@ void SNNetwork::worker_thread(std::string worker_id) { void SNNetwork::proxy_quit() { SN_LOG(debug, "Received quit command, shutting down proxy thread"); - int socket_linger = 5000; // milliseconds to try to send pending messages before shutting down (discarding pending) - assert(worker_threads.empty()); command.setsockopt(ZMQ_LINGER, 0); command.close(); @@ -442,33 +477,47 @@ void SNNetwork::proxy_quit() { control->close(); } workers.close(); - listener->setsockopt(ZMQ_LINGER, socket_linger); - listener.reset(); + if (listener) { + listener->setsockopt(ZMQ_LINGER, CLOSE_LINGER); + listener->close(); + } for (auto &r : remotes) - r->setsockopt(ZMQ_LINGER, socket_linger); + r.second.setsockopt(ZMQ_LINGER, CLOSE_LINGER); remotes.clear(); peers.clear(); SN_LOG(debug, "Proxy thread teardown complete"); } -std::pair, std::string> -SNNetwork::proxy_connect(const std::string &remote, const std::string &connect_hint, bool optional, std::chrono::milliseconds keep_alive) { - auto &peer = peers[remote]; +std::pair +SNNetwork::proxy_connect(const std::string &remote, const std::string &connect_hint, bool optional, bool incoming_only, std::chrono::milliseconds keep_alive) { + auto &peer = peers[remote]; // We may auto-vivify here, but that's okay; it'll get cleaned up in idle_expiry if no connection gets established - if (auto socket = peer.socket()) { + std::pair result = {nullptr, ""s}; + + bool outgoing = false; + if (peer.outgoing >= 0 && !incoming_only) { + result.first = &remotes[peer.outgoing].second; + outgoing = true; + } else if (!peer.incoming.empty() && listener) { + result.first = listener.get(); + result.second = peer.incoming; + } + + if (result.first) { SN_LOG(trace, "proxy asked to connect to " << as_hex(remote) << "; reusing existing connection"); - if (peer.idle_expiry < keep_alive) { - SN_LOG(debug, "updating existing peer connection idle expiry time from " << - peer.idle_expiry.count() << "ms to " << keep_alive.count() << "ms"); - peer.idle_expiry = keep_alive; + if (outgoing) { + if (peer.idle_expiry < keep_alive) { + SN_LOG(debug, "updating existing outgoing peer connection idle expiry time from " << + peer.idle_expiry.count() << "ms to " << keep_alive.count() << "ms"); + peer.idle_expiry = keep_alive; + } + peer.activity(); } - peer.activity(); - - return {socket, socket == listener ? peer.incoming_route : ""s}; - } else if (optional) { - SN_LOG(debug, "proxy asked for optional connection but not currently connected so cancelling connection attempt"); - return {}; + return result; + } else if (optional || incoming_only) { + SN_LOG(debug, "proxy asked for optional or incoming connection, but no appropriate connection exists so cancelling connection attempt"); + return result; } // No connection so establish a new one @@ -486,35 +535,37 @@ SNNetwork::proxy_connect(const std::string &remote, const std::string &connect_h if (addr.empty()) { SN_LOG(error, "quorumnet peer lookup failed for " << as_hex(remote)); - return {}; + return result; } } SN_LOG(debug, as_hex(pubkey) << " connecting to " << addr << " to reach " << as_hex(remote)); - auto socket = std::make_shared(context, zmq::socket_type::dealer); - socket->setsockopt(ZMQ_CURVE_SERVERKEY, remote.data(), remote.size()); - socket->setsockopt(ZMQ_CURVE_PUBLICKEY, pubkey.data(), pubkey.size()); - socket->setsockopt(ZMQ_CURVE_SECRETKEY, privkey.data(), privkey.size()); - socket->setsockopt(ZMQ_ZAP_DOMAIN, AUTH_DOMAIN_SN, sizeof(AUTH_DOMAIN_SN)-1); - socket->setsockopt(ZMQ_HANDSHAKE_IVL, SN_HANDSHAKE_TIME); - socket->setsockopt(ZMQ_MAXMSGSIZE, SN_ZMQ_MAX_MSG_SIZE); + zmq::socket_t socket{context, zmq::socket_type::dealer}; + socket.setsockopt(ZMQ_CURVE_SERVERKEY, remote.data(), remote.size()); + socket.setsockopt(ZMQ_CURVE_PUBLICKEY, pubkey.data(), pubkey.size()); + socket.setsockopt(ZMQ_CURVE_SECRETKEY, privkey.data(), privkey.size()); + socket.setsockopt(ZMQ_HANDSHAKE_IVL, SN_HANDSHAKE_TIME); + socket.setsockopt(ZMQ_MAXMSGSIZE, SN_ZMQ_MAX_MSG_SIZE); + // FIXME - do this? #if ZMQ_VERSION >= ZMQ_MAKE_VERSION (4, 3, 0) -// socket->setsockopt(ZMQ_ROUTING_ID, pubkey.data(), pubkey.size()); + socket.setsockopt(ZMQ_ROUTING_ID, pubkey.data(), pubkey.size()); #else -// socket->setsockopt(ZMQ_IDENTITY, pubkey.data(), pubkey.size()); + socket.setsockopt(ZMQ_IDENTITY, pubkey.data(), pubkey.size()); #endif - socket->connect(addr); + socket.connect(addr); peer.idle_expiry = keep_alive; - remotes.push_back(socket); - add_pollitem(*socket); - peer.outgoing = socket; + add_pollitem(socket); + peer.outgoing = remotes.size(); + remotes.emplace_back(remote, std::move(socket)); + peer.service_node = true; peer.activity(); - return {std::move(socket), ""s}; + result.first = &remotes.back().second; + return result; } -std::pair, std::string> SNNetwork::proxy_connect(bt_dict &&data) { +std::pair SNNetwork::proxy_connect(bt_dict &&data) { auto remote_pubkey = get(data.at("pubkey")); std::chrono::milliseconds keep_alive{get_int(data.at("keep-alive"))}; std::string hint; @@ -522,9 +573,9 @@ std::pair, std::string> SNNetwork::proxy_connect( if (hint_it != data.end()) hint = get(data.at("hint")); - bool optional = data.count("optional"); + bool optional = data.count("optional"), incoming = data.count("incoming"); - return proxy_connect(remote_pubkey, hint, optional, keep_alive); + return proxy_connect(remote_pubkey, hint, optional, incoming, keep_alive); } constexpr std::chrono::milliseconds SNNetwork::default_send_keep_alive; @@ -551,9 +602,9 @@ void SNNetwork::proxy_send(bt_dict &&data) { ? std::chrono::milliseconds(get_int(idle_it->second)) : default_send_keep_alive; - bool optional = data.count("optional"); + bool optional = data.count("optional"), incoming = data.count("incoming"); - auto sock_route = proxy_connect(remote_pubkey, hint, optional, keep_alive); + auto sock_route = proxy_connect(remote_pubkey, hint, optional, incoming, keep_alive); if (!sock_route.first) { if (optional) SN_LOG(debug, "Not sending: send is optional and no connection to " << as_hex(remote_pubkey) << " is currently established"); @@ -564,12 +615,11 @@ void SNNetwork::proxy_send(bt_dict &&data) { try { send_message_parts(*sock_route.first, build_send_parts(data, sock_route.second)); } catch (const zmq::error_t &e) { - if (e.num() == EHOSTUNREACH && sock_route.first == listener && !sock_route.second.empty()) { + if (e.num() == EHOSTUNREACH && sock_route.first == listener.get() && !sock_route.second.empty()) { // We *tried* to route via the incoming connection but it is no longer valid. Drop it, // establish a new connection, and try again. - auto peer = peers[remote_pubkey]; - peer.incoming.reset(); - peer.incoming_route.clear(); + auto &peer = peers[remote_pubkey]; + peer.incoming.clear(); // Don't worry about cleaning the map entry if outgoing is also < 0: that will happen at the next idle cleanup SN_LOG(debug, "Could not route back to SN " << as_hex(remote_pubkey) << " via listening socket; trying via new outgoing connection"); return proxy_send(std::move(data)); } @@ -580,6 +630,11 @@ void SNNetwork::proxy_send(bt_dict &&data) { void SNNetwork::proxy_reply(bt_dict &&data) { const auto &route = get(data.at("route")); assert(!route.empty()); + if (!listener) { + SN_LOG(error, "Internal error: proxy_reply called but that shouldn't be possible as we have no listener!"); + return; + } + try { send_message_parts(*listener, build_send_parts(data, route)); } catch (const zmq::error_t &err) { @@ -615,6 +670,8 @@ void SNNetwork::proxy_control_message(std::list parts) { idle_workers.clear(); } else if (cmd == "CONNECT") { proxy_connect(std::move(data)); + } else if (cmd == "DISCONNECT") { + proxy_disconnect(get(data.at("pubkey"))); } else if (cmd == "SEND") { SN_LOG(trace, "proxying message to " << as_hex(get(data.at("pubkey")))); proxy_send(std::move(data)); @@ -627,27 +684,56 @@ void SNNetwork::proxy_control_message(std::list parts) { } } -void SNNetwork::expire_idle_peers() { - for (auto &peer : peers) { - auto &info = peer.second; - auto idle = info.last_activity - std::chrono::steady_clock::now(); - if (idle <= info.idle_expiry) - continue; - auto outgoing = info.outgoing.lock(); - if (!outgoing) - continue; +auto SNNetwork::proxy_close_outgoing(decltype(peers)::iterator it) -> decltype(it) { + auto &peer = *it; + auto &info = peer.second; - SN_LOG(info, "Closing connection to " << as_hex(peer.first) << ": idle timeout reached"); + if (info.outgoing >= 0) { + remotes[info.outgoing].second.setsockopt(ZMQ_LINGER, CLOSE_LINGER); + pollitems.erase(pollitems.begin() + poll_remote_offset + info.outgoing); + remotes.erase(remotes.begin() + info.outgoing); + assert(remotes.size() == pollitems.size() + poll_remote_offset); - for (size_t i = 0; i < remotes.size(); i++) { - if (remotes[i] == outgoing) { - remotes[i]->setsockopt(ZMQ_LINGER, 0); - remotes.erase(remotes.begin() + i); - pollitems.erase(pollitems.begin() + poll_remote_offset + i); - assert(remotes.size() == pollitems.size() + poll_remote_offset); - break; + for (auto &p : peers) + if (p.second.outgoing > info.outgoing) + --p.second.outgoing; + + info.outgoing = -1; + } + + if (info.incoming.empty()) + // Neither incoming nor outgoing connections left, so erase the peer info + return peers.erase(it); + + return std::next(it); +} + +void SNNetwork::proxy_disconnect(const std::string &remote) { + auto it = peers.find(remote); + if (it == peers.end()) + return; + if (want_logs(LogLevel::debug)) { + if (it->second.outgoing >= 0) + SN_LOG(debug, "Closing outgoing connection to " << as_hex(it->first)); + } + proxy_close_outgoing(it); +} + +void SNNetwork::proxy_expire_idle_peers() { + for (auto it = peers.begin(); it != peers.end(); ) { + auto &info = it->second; + if (info.outgoing >= 0) { + auto idle = info.last_activity - std::chrono::steady_clock::now(); + if (idle <= info.idle_expiry) { + ++it; + continue; } + SN_LOG(info, "Closing outgoing connection to " << as_hex(it->first) << ": idle timeout reached"); } + + // Deliberately outside the above if: this *also* removes the peer from the map if if has + // neither an incoming or outgoing connection + it = proxy_close_outgoing(it); } } @@ -662,36 +748,39 @@ void SNNetwork::proxy_loop(const std::vector &bind) { spawn_worker("w1"); int next_worker_id = 2; - // Set up the public tcp listener(s): - auto &listen = *listener; - listen.setsockopt(ZMQ_ZAP_DOMAIN, AUTH_DOMAIN_SN, sizeof(AUTH_DOMAIN_SN)-1); - listen.setsockopt(ZMQ_CURVE_SERVER, 1); - listen.setsockopt(ZMQ_CURVE_PUBLICKEY, pubkey.data(), pubkey.size()); - listen.setsockopt(ZMQ_CURVE_SECRETKEY, privkey.data(), privkey.size()); - listen.setsockopt(ZMQ_MAXMSGSIZE, SN_ZMQ_MAX_MSG_SIZE); -// listen.setsockopt(ZMQ_ROUTER_HANDOVER, 1); - listen.setsockopt(ZMQ_ROUTER_MANDATORY, 1); - - for (const auto &b : bind) { - SN_LOG(info, "Quorumnet listening on " << b); - listen.bind(b); - } - - // Also add an internal connection to self so that calling code can avoid needing to - // special-case rare situations where we are supposed to talk to a quorum member that happens to - // be ourselves (which can happen, for example, with cross-quoum Blink communication) - self_listener->bind(SN_ADDR_SELF); - add_pollitem(command); add_pollitem(workers); add_pollitem(zap_auth); assert(pollitems.size() == poll_internal_size); - add_pollitem(*listener); - add_pollitem(*self_listener); + + if (listener) { + auto &l = *listener; + // Set up the public tcp listener(s): + l.setsockopt(ZMQ_ZAP_DOMAIN, AUTH_DOMAIN_SN, sizeof(AUTH_DOMAIN_SN)-1); + l.setsockopt(ZMQ_CURVE_SERVER, 1); + l.setsockopt(ZMQ_CURVE_PUBLICKEY, pubkey.data(), pubkey.size()); + l.setsockopt(ZMQ_CURVE_SECRETKEY, privkey.data(), privkey.size()); + l.setsockopt(ZMQ_MAXMSGSIZE, SN_ZMQ_MAX_MSG_SIZE); + l.setsockopt(ZMQ_ROUTER_HANDOVER, 1); + l.setsockopt(ZMQ_ROUTER_MANDATORY, 1); + + for (const auto &b : bind) { + SN_LOG(info, "Quorumnet listening on " << b); + l.bind(b); + } + + // Also add an internal connection to self so that calling code can avoid needing to + // special-case rare situations where we are supposed to talk to a quorum member that happens to + // be ourselves (which can happen, for example, with cross-quoum Blink communication) + l.bind(SN_ADDR_SELF); + + add_pollitem(l); + } + assert(pollitems.size() == poll_remote_offset); constexpr auto poll_timeout = 5000ms; // Maximum time we spend in each poll - constexpr auto timeout_check_interval = 2000ms; // Minimum time before for checking for connections to close since the last check + constexpr auto timeout_check_interval = 10000ms; // Minimum time before for checking for connections to close since the last check auto last_conn_timeout = std::chrono::steady_clock::now(); std::string waiting_for_worker; // If set contains the identify of a worker we just spawned but haven't received a READY signal from yet @@ -711,13 +800,14 @@ void SNNetwork::proxy_loop(const std::vector &bind) { // Otherwise, we just look for a control message or a worker coming back with a ready message. bool have_workers = idle_workers.size() > 0 || (worker_threads.size() < max_workers && waiting_for_worker.empty()); zmq::poll(pollitems.data(), have_workers ? pollitems.size() : poll_internal_size, poll_timeout); - SN_LOG(trace, "polled a waiting message"); + SN_LOG(trace, "processing control messages"); // Retrieve any waiting incoming control messages for (std::list parts; recv_message_parts(command, std::back_inserter(parts), zmq::recv_flags::dontwait); parts.clear()) { proxy_control_message(std::move(parts)); } + SN_LOG(trace, "processing worker messages"); // Process messages sent by workers for (std::list parts; recv_message_parts(workers, std::back_inserter(parts), zmq::recv_flags::dontwait); parts.clear()) { std::string route = pop_string(parts); @@ -740,6 +830,7 @@ void SNNetwork::proxy_loop(const std::vector &bind) { if (worker_threads.size() > max_workers) { // We have too many worker threads (possibly because we're shutting down) so // tell this worker to quit, and keep it non-idle. + SN_LOG(trace, "Telling worker " << route << " to quit"); route_control(workers, route, "QUIT"); } else { idle_workers.push_back(std::move(route)); @@ -758,6 +849,7 @@ void SNNetwork::proxy_loop(const std::vector &bind) { } } + SN_LOG(trace, "processing zap requests"); // Handle any zap authentication process_zap_requests(zap_auth); @@ -780,12 +872,14 @@ void SNNetwork::proxy_loop(const std::vector &bind) { continue; } + SN_LOG(trace, "processing incoming messages"); + // We round-robin connection queues for any pending messages (as long as we have enough // waiting workers), but we don't want a lot of earlier connection requests to starve // later request so each time through we continue from wherever we left off in the // previous queue. - const size_t num_sockets = remotes.size() + 2 /*listener + self*/; + const size_t num_sockets = remotes.size() + (listener ? 1 : 0); if (last_conn_index >= num_sockets) last_conn_index = 0; std::queue queue_index; @@ -795,84 +889,13 @@ void SNNetwork::proxy_loop(const std::vector &bind) { while (!idle_workers.empty() && !queue_index.empty()) { size_t i = queue_index.front(); queue_index.pop(); - auto &sock = *(i == 0 ? listener : i == 1 ? self_listener : remotes[i - 2]); + auto &sock = listener ? (i == 0 ? *listener : remotes[i - 1].second) : remotes[i].second; std::list parts; if (recv_message_parts(sock, std::back_inserter(parts), zmq::recv_flags::dontwait)) { last_conn_index = i; queue_index.push(i); // We just read one, but there might be more messages waiting so requeue it at the end - - // A messge to a worker takes one of the following forms: - // - // ["CONTROL"] -- for an internal proxy instruction such as "QUIT" - // ["PUBKEY", "CMD"] -- for a message send from an authenticated SN (ENCODED_BT_DICT is optional). - // ["", "ROUTE", "CMD"] -- for an incoming message from a non-SN node (i.e. not a SN quorum message) - // - // The latter two may be optionally followed by one frame containing a serialized bt_dict. - // - // The pubkey form supports sending a reply back to the given PUBKEY even if the - // original connection is closed -- a new connection to that SN will be - // established if required. The routed form only supports replying on the - // existing incoming connection (any attempted reply will be dropped if the - // connection no longer exists). - - std::string pubkey; - if (i == 1) { // Talking to ourself - pubkey = this->pubkey; - } else { - try { - const char *pubkey_hex = parts.back().gets("User-Id"); - auto len = std::strlen(pubkey_hex); - assert(len == 0 || len == 64); - if (len == 64) - pubkey = from_hex(pubkey_hex, pubkey_hex + 64); - } catch (...) {} // User-Id not set, i.e. no pubkey - } - - if (!pubkey.empty()) { - // SN message which means we want to stick the pubkey on the front. For a - // connection on the listener we also want to drop the first part (the - // return route) because SN replies have stronger routing. - if (i <= 1) // listener or self: discard the return route - parts.pop_front(); - - if (parts.size() < 1 || parts.size() > 2) { - SN_LOG(warn, "Invalid incoming message; expected 1-2 parts, not " << parts.size()); - continue; - } - - parts.emplace_front(pubkey.data(), pubkey.size()); - - auto it = peers.find(pubkey); - if (it != peers.end()) - it->second.activity(); - - } else { - // No pubkey (i.e. not a SN quorum message); this can only happen on an - // incoming connection on listener - assert(i == 0); - - // and, for such an incoming connection, the ZMQ router socket has already - // prepended a return path on the front: - auto incoming_parts = parts.size() - 1; - - if (incoming_parts < 1 || incoming_parts > 2) { - SN_LOG(warn, "Invalid incoming message: expected 1-2 parts, not " << incoming_parts); - continue; - } - - // Push an empty frame on the front to indicate a no-pubkey message - parts.emplace_front(); - } - - if (want_logs(LogLevel::trace)) { - const char *remote_addr = "(unknown)"; - try { remote_addr = parts.back().gets("Peer-Address"); } catch (...) {} - logger(LogLevel::trace, __FILE__, __LINE__, "Forwarding incoming message from " + - (pubkey.empty() ? "anonymous"s : as_hex(pubkey)) + " @ " + remote_addr + " to worker " + idle_workers.front()); - } - forward_to_worker(workers, std::move(idle_workers.front()), parts.begin(), parts.end()); - idle_workers.pop_front(); + proxy_to_worker(i, std::move(parts)); } } } @@ -883,13 +906,85 @@ void SNNetwork::proxy_loop(const std::vector &bind) { if (!idle_workers.empty()) { auto now = std::chrono::steady_clock::now(); if (now - last_conn_timeout >= timeout_check_interval) { - expire_idle_peers(); + SN_LOG(trace, "closing idle connections"); + proxy_expire_idle_peers(); last_conn_timeout = now; } } + + SN_LOG(trace, "done proxy loop"); } } +void SNNetwork::proxy_to_worker(size_t conn_index, std::list &&parts) { + // A message to a worker takes one of the following forms: + // + // ["CONTROL"] -- for an internal proxy instruction such as "QUIT" + // ["PUBKEY", "S", "CMD", ...] -- for a message relayed from an authenticated SN (... are optional bt_values). + // ["PUBKEY", "C", "CMD", ...] -- for an incoming message from a non-SN node (i.e. not a SN quorum message) + // + // The latter two may be optionally followed (the ...) by one or more frames containing serialized bt_values. + // + // Both forms support replying back to the given PUBKEY, but only the first form + // allows relaying back if the original connection is closed -- a new connection + // to that SN will be established if required. The "C" form only supports + // replying on the existing incoming connection (any attempted reply will be + // dropped if the connection no longer exists). + + std::string pubkey; + bool remote_is_sn; + if (conn_index > 0) { // Talking to a remote, we know the pubkey already, and is a SN. + pubkey = remotes[conn_index - 1].first; + remote_is_sn = 1; + } else { // Incoming; extract the pubkey from the message properties + try { + const char *pubkey_hex = parts.back().gets("User-Id"); + auto len = std::strlen(pubkey_hex); + assert(len == 66 && (pubkey_hex[0] == 'S' || pubkey_hex[0] == 'C') && pubkey_hex[1] == ':'); + pubkey = from_hex(pubkey_hex + 2, pubkey_hex + 66); + remote_is_sn = pubkey_hex[0] == 'S'; + } catch (...) { + SN_LOG(error, "Internal error: socket User-Id not set or invalid; dropping message"); + return; + } + } + assert(pubkey.size() == 32); + + auto &peer_info = peers[pubkey]; + peer_info.service_node |= remote_is_sn; + + if (conn_index < 1) { // incoming connection; pop off the route and update it if needed + auto route = pop_string(parts); + if (peer_info.incoming != route) + peer_info.incoming = route; + } else { // outgoing connection activity, pump the activity timer + peer_info.activity(); + } + + // We need at least a command: + if (parts.empty()) { + SN_LOG(warn, "Invalid empty incoming message; require at least a command. Dropping message."); + return; + } + + if (want_logs(LogLevel::trace)) { + const char *remote_addr = "(unknown)"; + try { remote_addr = parts.back().gets("Peer-Address"); } catch (...) {} + logger(LogLevel::trace, __FILE__, __LINE__, "Forwarding incoming message from " + + (pubkey.empty() ? "anonymous"s : as_hex(pubkey)) + " @ " + remote_addr + " to worker " + idle_workers.front()); + } + + // Prepend the [pubkey, [S|C]] for the worker + parts.emplace_front(peer_info.service_node ? "S" : "C", 1); + parts.push_front(create_message(std::move(pubkey))); + + // Prepend the worker name for the worker router to send it to the right worker + parts.emplace_front(idle_workers.front().data(), idle_workers.front().size()); + idle_workers.pop_front(); + + send_message_parts(workers, parts.begin(), parts.end()); +} + void SNNetwork::process_zap_requests(zmq::socket_t &zap_auth) { std::vector frames; for (frames.reserve(7); recv_message_parts(zap_auth, std::back_inserter(frames), zmq::recv_flags::dontwait); frames.clear()) { @@ -940,63 +1035,47 @@ void SNNetwork::process_zap_requests(zmq::socket_t &zap_auth) { std::string &status_code = response_vals[2], &status_text = response_vals[3]; if (frames.size() < 6 || view(frames[0]) != "1.0") { - SN_LOG(error, "Bad ZAP authentication request: version != 1.0"); + SN_LOG(error, "Bad ZAP authentication request: version != 1.0 or invalid ZAP message parts"); status_code = "500"; status_text = "Internal error: invalid auth request"; } - else if (view(frames[2]) == AUTH_DOMAIN_SN) { - // An auth request - auto mech = view(frames[5]); - if (mech != "CURVE") { - SN_LOG(error, "Bad ZAP authentication request: expected CURVE authentication"); + else if (frames.size() != 7 || view(frames[5]) != "CURVE") { + SN_LOG(error, "Bad ZAP authentication request: invalid CURVE authentication request"); + status_code = "500"; + status_text = "Invalid CURVE authentication request\n"; + } + else if (frames[6].size() != 32) { + SN_LOG(error, "Bad ZAP authentication request: invalid request pubkey"); + status_code = "500"; + status_text = "Invalid public key size for CURVE authentication"; + } + else { + auto domain = view(frames[2]); + if (domain != AUTH_DOMAIN_SN) { + SN_LOG(error, "Bad ZAP authentication request: invalid auth domain '" << domain << "'"); status_code = "400"; - status_text = "Invalid quorum connection authentication mechanism: " + (std::string) mech; - } - else if (frames.size() != 7) { - SN_LOG(error, "Bad ZAP authentication request: invalid request message size"); - status_code = "500"; - status_text = "Invalid CURVE authentication request\n"; - } - else if (frames[6].size() != 32) { - SN_LOG(error, "Bad ZAP authentication request: invalid request pubkey"); - status_code = "500"; - status_text = "Invalid public key size for CURVE authentication"; - } - else { - std::string ip{view(frames[3])}, pubkey{view(frames[6])}; - if (allow_connection(ip, pubkey)) { - SN_LOG(info, "Successfully authenticated incoming connection from " << as_hex(pubkey) << " at " << ip); + status_text = "Unknown authentication domain: " + std::string{domain}; + } else { + std::string ip{view(frames[3])}, pubkey{view(frames[6])}, pubkey_hex{as_hex(pubkey)}; + auto result = allow_connection(ip, pubkey); + bool sn = result == allow::service_node; + if (sn || result == allow::client) { + SN_LOG(info, "Successfully " << + (sn ? "authenticated incoming service node" : "accepted incoming non-SN client") << + " connection from " << pubkey_hex << " at " << ip); status_code = "200"; status_text = ""; - response_vals[4 /*user-id*/] = as_hex(pubkey); // ZMQ `gets` requires a null-terminated value + // Set a user-id to the pubkey prefixed with "C:" or "S:" to indicate how we + // recognized the pubkey. + response_vals[4 /*user-id*/] = (sn ? "S:" : "C:") + pubkey_hex; // hex because zmq message property strings are null-terminated } else { - SN_LOG(info, "Authentication failed for incoming connection from " << as_hex(pubkey) << " at " << ip); + SN_LOG(info, "Access denied for incoming " << (sn ? "service node" : "non-SN client") << + " connection from " << pubkey_hex << " at " << ip); status_code = "400"; status_text = "Access denied"; } } } - else if (view(frames[2]) == AUTH_DOMAIN_CLIENT) { - std::string ip{view(frames[3])}; - auto mech = view(frames[5]); - if (mech != "NULL") { - status_code = "400"; - status_text = "Client connections require NULL authentication, not " + (std::string) mech; - } - else if (allow_connection(ip, "")) { - SN_LOG(info, "Accepted incoming client connection from " << ip); - status_code = "200"; - status_text = ""; - } else { - SN_LOG(info, "Rejected incoming client connection rejected from " << ip); - status_code = "400"; - status_text = "Access denied"; - } - } - else { - status_code = "400"; - status_text = "Unknown authentication domain: " + std::string{view(frames[2])}; - } SN_LOG(trace, "ZAP request result: " << status_code << " " << status_text); @@ -1017,13 +1096,6 @@ void SNNetwork::connect(const std::string &pubkey, std::chrono::milliseconds kee detail::send_control(get_control_socket(), "CONNECT", {{"pubkey",pubkey}, {"keep-alive",keep_alive.count()}, {"hint",hint}}); } -std::shared_ptr SNNetwork::peer_info::socket() { - auto sock = outgoing.lock(); - if (!sock) - sock = incoming.lock(); - return sock; -} - } diff --git a/src/quorumnet/sn_network.h b/src/quorumnet/sn_network.h index 22424f184..f7b39f999 100644 --- a/src/quorumnet/sn_network.h +++ b/src/quorumnet/sn_network.h @@ -62,19 +62,19 @@ public: /// return an empty string for an invalid or unknown pubkey or one without a known address. using LookupFunc = std::function; + enum class allow { denied, client, service_node }; /// Callback type invoked to determine whether the given ip and pubkey are allowed to connect to - /// us. This will be called in two contexts: + /// us as a SN, client, or not at all. /// - /// - if the connection is from another SN for quorum-related duties, the pubkey will be set - /// to the verified pubkey of the remote service node. - /// - otherwise, for a client connection (for example, a regular node connecting to submit a - /// blink transaction to a blink quorum) the pubkey will be empty. - /// - /// @param ip - the ip address of the incoming connection + /// @param ip - the ip address of the incoming connection; will be empty if called to attempt to + /// "upgrade" the permission of an existing non-SN connection. /// @param pubkey - the curve25519 pubkey (which is calculated from the SN ed25519 pubkey) of /// the connecting service node (32 byte string), or an empty string if this is a client /// connection without remote SN authentication. - using AllowFunc = std::function; + /// @returns an `allow` enum value: `denied` if the connection is not allowed, `client` if the + /// connection is allowed as a client (i.e. not for SN-to-SN commands), `service_node` if the + /// connection is a valid SN. + using AllowFunc = std::function; /// Function pointer to ask whether a log of the given level is wanted. If it returns true the /// log message will be built and then passed to Log. @@ -100,28 +100,23 @@ public: public: std::string command; ///< The command name std::vector data; ///< The provided command data parts, if any. - const std::string pubkey; ///< The originator pubkey (32 bytes), if from an authenticated service node; empty for a non-SN incoming message. - const std::string route; ///< Opaque routing string used to route a reply back to the correct place when `pubkey` is empty. + const std::string pubkey; ///< The originator pubkey (32 bytes) + const bool sn; ///< True if the pubkey is from a SN, meaning we can/should reconnect to it if necessary /// Constructor - message(SNNetwork &net, std::string command, std::string pubkey, std::string route) - : net{net}, command{std::move(command)}, pubkey{std::move(pubkey)}, route{std::move(route)} { - assert(this->pubkey.empty() ? !this->route.empty() : this->pubkey.size() == 32); + message(SNNetwork &net, std::string command, std::string pubkey, bool sn) + : net{net}, command{std::move(command)}, pubkey{std::move(pubkey)}, sn{sn} { + assert(this->pubkey.size() == 32); } - /// True if this message is from a service node (i.e. pubkey is set) - bool from_sn() const { return !pubkey.empty(); } - - /// Sends a reply. Arguments are forward appropriately to send() or reply_incoming(). For - /// SN messages (i.e. where `from_sn()` is true) this is a "strong" reply by default in - /// that the proxy will establish a new connection to the SN if no longer connected. For - /// non-SN messages the reply will be attempted using the available routing information, but - /// if the connection has already been closed the reply will be dropped. + /// Sends a reply. Arguments are forward to send() but with send_option::optional{} added + /// if the originator is not a SN. For SN messages (i.e. where `sn` is true) this is a + /// "strong" reply by default in that the proxy will establish a new connection to the SN if + /// no longer connected. For non-SN messages the reply will be attempted using the + /// available routing information, but if the connection has already been closed the reply + /// will be dropped. template - void reply(const std::string &command, Args &&...args) { - if (from_sn()) net.send(pubkey, command, std::forward(args)...); - else net.reply_incoming(route, command, std::forward(args)...); - } + void reply(const std::string &command, Args &&...args); }; /// Opaque pointer sent to the callbacks, to allow for including arbitrary state data (for @@ -135,6 +130,9 @@ private: /// A unique id for this SNNetwork, assigned in a thread-safe manner during construction. const int object_id; + /// The keypair of this SN, either provided or generated during construction + std::string pubkey, privkey; + /// The thread in which most of the intermediate work happens (handling external connections /// and proxying requests between them to worker threads) std::thread proxy_thread; @@ -156,11 +154,9 @@ private: /// The lookup function that tells us where to connect to a peer LookupFunc peer_lookup; - /// Our listening socket for public connections - std::shared_ptr listener = std::make_shared(context, zmq::socket_type::router); - /// Our listening socket for ourselves (so that we can just "connect" and talk to ourselves - /// without worrying about special casing it). - std::shared_ptr self_listener = std::make_shared(context, zmq::socket_type::router); + /// Our listening socket for public connections, or null for a remote-only + /// (TODO: change this to std::optional once we use C++17). + std::unique_ptr listener; /// Callback to see whether the incoming connection is allowed AllowFunc allow_connection; @@ -173,18 +169,18 @@ private: /// Info about a peer's established connection to us. Note that "established" means both /// connected and authenticated. struct peer_info { - /// Will be set to `listener` if we have an established incoming connection (but note that - /// the connection might no longer be valid) and empty otherwise. - std::weak_ptr incoming; + /// True if we've authenticated this peer as a service node. (Note that new outgoing + /// connections are *always* expected to go to service nodes and will update this to true). + bool service_node = false; - /// FIXME: neither the above nor below are currently being set on an incoming connection! + /// Will be set to a non-empty routing prefix (which needs to be prefixed out outgoing + /// messages) if we have (or at least recently had) an established incoming connection with + /// this peer. Will be empty if there is no incoming connection. + std::string incoming; - /// The routing prefix needed to reply to the connection on the incoming socket. - std::string incoming_route; - - /// Our outgoing socket, if we have an established outgoing connection to this peer. The - /// owning pointer is in `remotes`. - std::weak_ptr outgoing; + /// The index in `remotes` if we have an established outgoing connection to this peer, -1 if + /// we have no outgoing connection to this peer. + int outgoing = -1; /// The last time we sent or received a message (or had some other relevant activity) with /// this peer. Used for closing outgoing connections that have reached an inactivity expiry @@ -196,19 +192,17 @@ private: /// After more than this much activity we will close an idle connection std::chrono::milliseconds idle_expiry; - - /// Returns a socket to talk to the given peer, if we have one. If we don't, the returned - /// pointer will be empty. If both outgoing and incoming connections are available the - /// outgoing connection is preferred. - std::shared_ptr socket(); }; + + struct pk_hash { size_t operator()(const std::string &pubkey) const { size_t h; std::memcpy(&h, pubkey.data(), sizeof(h)); return h; } }; + /// Currently peer connections, pubkey -> peer_info - std::unordered_map peers; + std::unordered_map peers; /// different polling sockets the proxy handler polls: this always contains some internal /// sockets for inter-thread communication followed by listener socket and a pollitem for every /// (outgoing) remote socket in `remotes`. This must be in a sequential vector because of zmq - /// requirements (otherwise it would be far nicer to not have to synchronize these two vectors). + /// requirements (otherwise it would be far nicer to not have to synchronize the two vectors). std::vector pollitems; /// Properly adds a socket to poll for input to pollitems @@ -218,13 +212,13 @@ private: static constexpr size_t poll_internal_size = 3; /// The pollitems location corresponding to `remotes[0]`. - static constexpr size_t poll_remote_offset = poll_internal_size + 2; // +2 because we also have the incoming listener and self sockets + const size_t poll_remote_offset; // will be poll_internal_size + 1 for a full listener (the +1 is the listening socket); poll_internal_size for a remote-only - /// The outgoing remote connections we currently have open. Note that they are generally - /// accessed via the weak_ptr inside the `peers` element. Each element [i] here corresponds to - /// an the pollitem_t at pollitems[i+1+poll_internal_size]. (Ideally we'd use one structure, - /// but zmq requires the pollitems be in contiguous storage). - std::vector> remotes; + /// The outgoing remote connections we currently have open along with the remote pubkeys. Note + /// that the sockets here are generally accessed via the weak_ptr inside the `peers` element. + /// Each element [i] here corresponds to an the pollitem_t at pollitems[i+1+poll_internal_size]. + /// (Ideally we'd use one structure, but zmq requires the pollitems be in contiguous storage). + std::vector> remotes; /// Socket we listen on to receive control messages in the proxy thread. Each thread has its own /// internal "control" connection (returned by `get_control_socket()`) to this socket used to @@ -254,6 +248,9 @@ private: /// Does the proxying work void proxy_loop(const std::vector &bind); + /// Forwards an incoming message to an idle worker, removing the idle worker from the queue + void proxy_to_worker(size_t conn_index, std::list &&parts); + /// proxy thread command handlers for commands sent from the outer object QUIT. This doesn't /// get called immediately on a QUIT command: the QUIT commands tells workers to quit, then this /// gets called after all works have done so. @@ -261,11 +258,16 @@ private: /// Common connection implementation used by proxy_connect/proxy_send. Returns the socket /// and, if a routing prefix is needed, the required prefix (or an empty string if not needed). - std::pair, std::string> proxy_connect(const std::string &pubkey, const std::string &connect_hint, bool optional, std::chrono::milliseconds keep_alive); + /// For an optional connect that fail, returns nullptr for the socket. + std::pair proxy_connect(const std::string &pubkey, const std::string &connect_hint, bool optional, bool incoming_only, std::chrono::milliseconds keep_alive); /// CONNECT command telling us to connect to a new pubkey. Returns the socket (which could be /// existing or a new one). - std::pair, std::string> proxy_connect(bt_dict &&data); + std::pair proxy_connect(bt_dict &&data); + + /// DISCONNECT command telling us to disconnect out remote connection to the given pubkey (if we + /// have one). + void proxy_disconnect(const std::string &pubkey); /// SEND command. Does a connect first, if necessary. void proxy_send(bt_dict &&data); @@ -284,7 +286,12 @@ private: /// Closing any idle connections that have outlived their idle time. Note that this only /// affects outgoing connections; incomings connections are the responsibility of the other end. - void expire_idle_peers(); + void proxy_expire_idle_peers(); + + /// Closes an outgoing connection immediately, updates internal variables appropriately. + /// Returns the next iterator (the original may or may not be removed from peers, depending on + /// whether or not it also has an active incoming connection). + decltype(peers)::iterator proxy_close_outgoing(decltype(peers)::iterator it); /// End of proxy-specific members @@ -301,6 +308,9 @@ private: static std::unordered_map, bool>> commands; static bool commands_mutable; + /// Starts up the proxy thread; called during construction + void launch_proxy_thread(const std::vector &bind); + public: /** * Constructs a SNNetwork connection listening on the given bind string. @@ -334,6 +344,17 @@ public: WriteLog logger = [](LogLevel, const char *f, int line, std::string msg) { std::cerr << f << ':' << line << ": " << msg << std::endl; }, unsigned int max_workers = std::thread::hardware_concurrency()); + /** Constructs a SNNetwork that does not listen but can be used for connecting to remote + * listening service nodes, for example to submit blink transactions to service nodes. It + * generates a non-persistant x25519 key pair on startup (for encrypted communication with + * peers). + */ + SNNetwork(LookupFunc peer_lookup, + AllowFunc allow_connection, + WantLog want_log = [](LogLevel l) { return l >= LogLevel::warn; }, + WriteLog logger = [](LogLevel, const char *f, int line, std::string msg) { std::cerr << f << ':' << line << ": " << msg << std::endl; }, + unsigned int max_workers = std::thread::hardware_concurrency()); + /** * Destructor; instructs the proxy to quit. The proxy tells all workers to quit, waits for them * to quit and rejoins the threads then quits itself. The outer thread (where the destructor is @@ -392,21 +413,15 @@ public: template void send(const std::string &pubkey, const std::string &cmd, const T &...opts); - /** Queue a message to be replied to the given incoming route, if still connected. This method - * is typically invoked indirectly by calling `message.reply(...)` which either calls `send()` - * or this message, depending on the original source of the message. - */ - template - void reply_incoming(const std::string &route, const std::string &cmd, const T &...opts); - /** The keep-alive time for a send() that results in a new connection. To use a longer * keep-alive to a host call `connect()` first with the desired keep-alive time or pass the - * send_options::keep_alive + * send_option::keep_alive */ static constexpr std::chrono::milliseconds default_send_keep_alive{15000}; /// The key pair this SN was created with - const std::string pubkey, privkey; + const std::string &get_pubkey() const { return pubkey; } + const std::string &get_privkey() const { return privkey; } /** * Registers a quorum command that may be invoked by authenticated SN connections but not @@ -433,6 +448,16 @@ public: /// Namespace for options to the send() method namespace send_option { + +/// `serialized` lets you serialize once when sending the same data to many peers by constructing a +/// single serialized option and passing it repeatedly rather than needing to reserialize on each +/// send. +struct serialized { + std::string data; + template + serialized(const T &arg) : data{quorumnet::bt_serialize(arg)} {} +}; + /// Specifies a connection hint when passed in to send(). If there is no current connection to the /// peer then the hint is used to save a call to the LookupFunc to get the connection location. /// (Note that there is no guarantee that the given hint will be used or that a LookupFunc call will @@ -446,6 +471,10 @@ struct hint { /// otherwise drops the message. struct optional {}; +/// Specifies that the message should be sent only if it can be sent on an existing incoming socket, +/// and dropped otherwise. +struct incoming {}; + /// Specifies the idle timeout for the connection - if a new or existing outgoing connection is used /// for the send and its current idle timeout setting is less than this value then it is updated. struct keep_alive { @@ -467,6 +496,11 @@ void apply_send_option(bt_list &parts, bt_dict &, const T &arg) { parts.push_back(quorumnet::bt_serialize(arg)); } +/// `serialized` specialization: lets you serialize once when sending the same data to many peers +template <> inline void apply_send_option(bt_list &parts, bt_dict &, const send_option::serialized &serialized) { + parts.push_back(serialized.data); +} + /// `hint` specialization: sets the hint in the control data template <> inline void apply_send_option(bt_list &, bt_dict &control_data, const send_option::hint &hint) { control_data["hint"] = hint.connect_hint; @@ -477,6 +511,11 @@ template <> inline void apply_send_option(bt_list &, bt_dict &control_data, cons control_data["optional"] = 1; } +/// `incoming` specialization: sets the optional flag in the control data +template <> inline void apply_send_option(bt_list &, bt_dict &control_data, const send_option::incoming &) { + control_data["incoming"] = 1; +} + /// `keep_alive` specialization: increases the outgoing socket idle timeout (if shorter) template <> inline void apply_send_option(bt_list &, bt_dict &control_data, const send_option::keep_alive &timeout) { control_data["keep-alive"] = timeout.time.count(); @@ -507,14 +546,12 @@ void SNNetwork::send(const std::string &pubkey, const std::string &cmd, const T detail::send_control(get_control_socket(), "SEND", control_data); } -template -void SNNetwork::reply_incoming(const std::string &route, const std::string &cmd, const T &...opts) { - bt_dict control_data = detail::send_control_data(cmd, opts...); - control_data["route"] = route; - detail::send_control(get_control_socket(), "REPLY", control_data); +template +void SNNetwork::message::reply(const std::string &command, Args &&...args) { + if (sn) net.send(pubkey, command, std::forward(args)...); + else net.send(pubkey, command, send_option::optional{}, std::forward(args)...); } - // Creates a hex string from a character sequence. template std::string as_hex(It begin, It end) { @@ -533,7 +570,9 @@ std::string as_hex(It begin, It end) { template inline std::string as_hex(const String &s) { - return as_hex(s.begin(), s.end()); + using std::begin; + using std::end; + return as_hex(begin(s), end(s)); } diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 632d6448d..c138e372a 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -841,6 +841,27 @@ namespace cryptonote } res.sanity_check_failed = false; + if (req.blink) + { + using namespace std::chrono_literals; + auto future = m_core.handle_blink_tx(tx_blob); + auto status = future.wait_for(10s); + if (status != std::future_status::ready) { + res.status = "Failed"; + res.reason = "Blink quorum timeout"; + return true; + } + + auto result = future.get(); + if (result.first == blink_result::accepted) { + res.status = CORE_RPC_STATUS_OK; + } else { + res.status = "Failed"; + res.reason = !result.second.empty() ? result.second : result.first == blink_result::timeout ? "Blink quorum timeout" : "Transaction rejected by blink quorum"; + } + return true; + } + cryptonote_connection_context fake_context{}; tx_verification_context tvc{}; if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, req.do_not_relay) || tvc.m_verifivation_failed) diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 0a07feab8..d178eb466 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -341,11 +341,13 @@ namespace cryptonote std::string address; std::string view_key; std::string tx; + bool blink; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(address) KV_SERIALIZE(view_key) KV_SERIALIZE(tx) + KV_SERIALIZE_OPT(blink, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init request; @@ -699,13 +701,15 @@ namespace cryptonote struct request_t { std::string tx_as_hex; // Full transaction information as hexidecimal string. - bool do_not_relay; // (Optional: Default false) Stop relaying transaction to other nodes. + bool do_not_relay; // (Optional: Default false) Stop relaying transaction to other nodes. Ignored if `blink` is true. bool do_sanity_checks; // (Optional: Default true) Verify TX params have sane values. + bool blink; // (Optional: Default false) Submit this as a blink tx rather than into the mempool. BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_as_hex) KV_SERIALIZE_OPT(do_not_relay, false) KV_SERIALIZE_OPT(do_sanity_checks, true) + KV_SERIALIZE_OPT(blink, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init request; diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index a8c8a8117..2e7793994 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -135,11 +135,6 @@ typedef cryptonote::simple_wallet sw; } \ } while(0) -enum TransferType { - Transfer, - TransferLocked, -}; - namespace { const auto arg_wallet_file = wallet_args::arg_wallet_file(); @@ -172,12 +167,13 @@ namespace const char* USAGE_INCOMING_TRANSFERS("incoming_transfers [available|unavailable] [verbose] [uses] [index=[,[,...]]]"); const char* USAGE_PAYMENTS("payments [ ... ]"); const char* USAGE_PAYMENT_ID("payment_id"); - const char* USAGE_TRANSFER("transfer [index=[,,...]] [] ( |
) []"); + const char* USAGE_TRANSFER("transfer [index=[,,...]] [blink|] ( |
) []"); + const char* USAGE_BLINK("blink [index=[,,...]] ( |
)"); const char* USAGE_LOCKED_TRANSFER("locked_transfer [index=[,,...]] [] ( | ) []"); const char* USAGE_LOCKED_SWEEP_ALL("locked_sweep_all [index=[,,...]] []
[]"); - const char* USAGE_SWEEP_ALL("sweep_all [index=[,,...]] [] [outputs=]
[] [use_v1_tx]"); - const char* USAGE_SWEEP_BELOW("sweep_below [index=[,,...]] []
[]"); - const char* USAGE_SWEEP_SINGLE("sweep_single [] [outputs=]
[]"); + const char* USAGE_SWEEP_ALL("sweep_all [index=[,,...]] [blink|] [outputs=]
[] [use_v1_tx]"); + const char* USAGE_SWEEP_BELOW("sweep_below [index=[,,...]] [blink|]
[]"); + const char* USAGE_SWEEP_SINGLE("sweep_single [blink|] [outputs=]
[]"); const char* USAGE_SIGN_TRANSFER("sign_transfer [export_raw]"); const char* USAGE_SET_LOG("set_log |{+,-,}"); const char* USAGE_ACCOUNT("account\n" @@ -1549,10 +1545,12 @@ bool simple_wallet::submit_multisig_main(const std::vector &args, b return false; } + constexpr bool FIXME_blink = false; // Blink not supported yet for multisig wallets + // actually commit the transactions for (auto &ptx: txs.m_ptx) { - m_wallet->commit_tx(ptx); + m_wallet->commit_tx(ptx, FIXME_blink); success_msg_writer(true) << tr("Transaction successfully submitted, transaction ") << get_transaction_hash(ptx.tx) << ENDL << tr("You can check its status by using the `show_transfers` command."); } @@ -2729,11 +2727,11 @@ simple_wallet::simple_wallet() tr("Show the blockchain height.")); m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), tr(USAGE_TRANSFER), - tr("Transfer to
. If the parameter \"index=[,,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. Multiple payments can be made at once by adding etcetera (before the payment ID, if it's included)")); + tr("Transfer to
. If the parameter \"index=[,,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. is the priority of the transaction, or \"blink\" for an instant transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. Multiple payments can be made at once by adding et cetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_transfer", boost::bind(&simple_wallet::locked_transfer, this, _1), tr(USAGE_LOCKED_TRANSFER), - tr("Transfer to
and lock it for (max. 1000000). If the parameter \"index=[,,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. Multiple payments can be made at once by adding URI_2 or etcetera (before the payment ID, if it's included)")); + tr("Transfer to
and lock it for (max. 1000000). If the parameter \"index=[,,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. Multiple payments can be made at once by adding URI_2 or et cetera (before the )")); m_cmd_binder.set_handler("locked_sweep_all", boost::bind(&simple_wallet::locked_sweep_all, this, _1), tr(USAGE_LOCKED_SWEEP_ALL), @@ -5612,7 +5610,7 @@ static bool locked_blocks_arg_valid(const std::string& arg, uint64_t& duration) } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::transfer_main(int transfer_type, const std::vector &args_, bool called_by_mms) +bool simple_wallet::transfer_main(Transfer transfer_type, const std::vector &args_, bool called_by_mms) { // "transfer [index=[,,...]] []
[]" if (!try_connect_to_daemon()) @@ -5638,7 +5636,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vectoradjust_priority(priority); - const size_t min_args = (transfer_type == TransferLocked) ? 2 : 1; + const size_t min_args = (transfer_type == Transfer::Locked) ? 2 : 1; if(local_args.size() < min_args) { fail_msg_writer() << tr("wrong number of arguments"); @@ -5671,8 +5669,14 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector ptx_vector; uint64_t bc_height, unlock_block = 0; std::string err; - switch (transfer_type) + if (transfer_type == Transfer::Locked) { - case TransferLocked: - bc_height = get_daemon_blockchain_height(err); - if (!err.empty()) - { - fail_msg_writer() << tr("failed to get blockchain height: ") << err; - return false; - } - unlock_block = bc_height + locked_blocks; - ptx_vector = m_wallet->create_transactions_2(dsts, CRYPTONOTE_DEFAULT_TX_MIXIN, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); - break; - default: - LOG_ERROR("Unknown transfer method, using default"); - /* FALLTHRU */ - case Transfer: - ptx_vector = m_wallet->create_transactions_2(dsts, CRYPTONOTE_DEFAULT_TX_MIXIN, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); - break; + bc_height = get_daemon_blockchain_height(err); + if (!err.empty()) + { + fail_msg_writer() << tr("failed to get blockchain height: ") << err; + return false; + } + unlock_block = bc_height + locked_blocks; } + ptx_vector = m_wallet->create_transactions_2(dsts, CRYPTONOTE_DEFAULT_TX_MIXIN, unlock_block, priority, extra, m_current_subaddress_account, subaddr_indices); + if (ptx_vector.empty()) { fail_msg_writer() << tr("No outputs found, or daemon is not ready"); @@ -5829,7 +5826,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vectorconfirm_backlog()) + if (m_wallet->confirm_backlog() && priority != tools::wallet2::BLINK_PRIORITY) { std::stringstream prompt; double worst_fee_per_byte = std::numeric_limits::max(); @@ -5924,7 +5921,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector &args_) { - transfer_main(Transfer, args_, false); - return true; + return transfer_main(Transfer::Normal, args_, false); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::locked_transfer(const std::vector &args_) { - transfer_main(TransferLocked, args_, false); - return true; + return transfer_main(Transfer::Locked, args_, false); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::locked_sweep_all(const std::vector &args_) { - return sweep_main(0, true, args_); + return sweep_main(0, Transfer::Locked, args_); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::register_service_node(const std::vector &args_) @@ -6087,7 +6082,7 @@ bool simple_wallet::register_service_node(const std::vector &args_) try { std::vector ptx_vector = {result.ptx}; - if (!sweep_main_internal(sweep_type_t::register_stake, ptx_vector, info)) + if (!sweep_main_internal(sweep_type_t::register_stake, ptx_vector, info, false /* don't blink */)) { fail_msg_writer() << tr("Sending register transaction failed"); return true; @@ -6202,7 +6197,7 @@ bool simple_wallet::stake(const std::vector &args_) tools::msg_writer() << stake_result.msg; std::vector ptx_vector = {stake_result.ptx}; - if (!sweep_main_internal(sweep_type_t::stake, ptx_vector, info)) + if (!sweep_main_internal(sweep_type_t::stake, ptx_vector, info, false /* don't blink */)) { fail_msg_writer() << tr("Sending stake transaction failed"); return true; @@ -6283,7 +6278,7 @@ bool simple_wallet::request_stake_unlock(const std::vector &args_) try { - commit_or_save(ptx_vector, m_do_not_relay); + commit_or_save(ptx_vector, m_do_not_relay, false /* don't blink */); } catch (const std::exception &e) { @@ -6525,7 +6520,7 @@ bool simple_wallet::sweep_unmixable(const std::vector &args_) } else { - commit_or_save(ptx_vector, m_do_not_relay); + commit_or_save(ptx_vector, m_do_not_relay, false /* don't blink */); } } catch (const std::exception &e) @@ -6541,7 +6536,7 @@ bool simple_wallet::sweep_unmixable(const std::vector &args_) return true; } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::sweep_main_internal(sweep_type_t sweep_type, std::vector &ptx_vector, cryptonote::address_parse_info const &dest) +bool simple_wallet::sweep_main_internal(sweep_type_t sweep_type, std::vector &ptx_vector, cryptonote::address_parse_info const &dest, bool blink) { if ((sweep_type == sweep_type_t::stake || sweep_type == sweep_type_t::register_stake) && ptx_vector.size() > 1) { @@ -6651,7 +6646,7 @@ bool simple_wallet::sweep_main_internal(sweep_type_t sweep_type, std::vectoradjust_priority(priority); uint64_t unlock_block = 0; - if (locked) { + if (transfer_type == Transfer::Locked) { + if (priority == tools::wallet2::BLINK_PRIORITY) { + fail_msg_writer() << tr("blink priority cannot be used for locked transfers"); + return false; + } uint64_t locked_blocks = 0; if (local_args.size() < 2) { @@ -6860,8 +6859,8 @@ bool simple_wallet::sweep_main(uint64_t below, Transfer transfer_type, const std SCOPED_WALLET_UNLOCK(); try { - auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, outputs, CRYPTONOTE_DEFAULT_TX_MIXIN, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, false, sweep_style); - sweep_main_internal(sweep_type_t::all_or_below, ptx_vector, info); + auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, outputs, CRYPTONOTE_DEFAULT_TX_MIXIN, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, false); + sweep_main_internal(sweep_type_t::all_or_below, ptx_vector, info, priority == tools::wallet2::BLINK_PRIORITY); } catch (const std::exception &e) { @@ -6996,7 +6995,7 @@ bool simple_wallet::sweep_single(const std::vector &args_) { // figure out what tx will be necessary auto ptx_vector = m_wallet->create_transactions_single(ki, info.address, info.is_subaddress, outputs, CRYPTONOTE_DEFAULT_TX_MIXIN, 0 /* unlock_time */, priority, extra); - sweep_main_internal(sweep_type_t::single, ptx_vector, info); + sweep_main_internal(sweep_type_t::single, ptx_vector, info, priority == tools::wallet2::BLINK_PRIORITY); } catch (const std::exception& e) { @@ -7029,7 +7028,7 @@ bool simple_wallet::sweep_below(const std::vector &args_) fail_msg_writer() << tr("invalid amount threshold"); return true; } - return sweep_main(below, false, std::vector(++args_.begin(), args_.end())); + return sweep_main(below, Transfer::Normal, std::vector(++args_.begin(), args_.end())); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::accept_loaded_tx(const std::function get_num_txes, const std::function &get_tx, const std::string &extra_message) @@ -7279,7 +7278,10 @@ bool simple_wallet::submit_transfer(const std::vector &args_) return true; } - commit_or_save(ptx_vector, false); + // FIXME: store the blink status in the signed_loki_tx somehow? + constexpr bool FIXME_blink = false; + + commit_or_save(ptx_vector, false, FIXME_blink); } catch (const std::exception& e) { @@ -9382,7 +9384,7 @@ void simple_wallet::interrupt() } } //---------------------------------------------------------------------------------------------------- -void simple_wallet::commit_or_save(std::vector& ptx_vector, bool do_not_relay) +void simple_wallet::commit_or_save(std::vector& ptx_vector, bool do_not_relay, bool blink) { size_t i = 0; std::string msg_buf; // NOTE(loki): Buffer output so integration tests read the entire output @@ -9414,7 +9416,7 @@ void simple_wallet::commit_or_save(std::vector& ptx_ } else { - m_wallet->commit_tx(ptx); + m_wallet->commit_tx(ptx, blink); msg_buf += tr("Transaction successfully submitted, transaction <"); msg_buf += epee::string_tools::pod_to_hex(txid); msg_buf += ">\n"; @@ -10111,7 +10113,7 @@ void simple_wallet::mms_sync(const std::vector &args) void simple_wallet::mms_transfer(const std::vector &args) { // It's too complicated to check any arguments here, just let 'transfer_main' do the whole job - transfer_main(Transfer, args, true); + transfer_main(Transfer::Normal, args, true); } void simple_wallet::mms_delete(const std::vector &args) diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index c633f96b1..cee8a5e4c 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -61,6 +61,11 @@ constexpr const char MONERO_DONATION_ADDR[] = "44AFFq5kSiGBoZ4NMDwYtN18obc8AemS3 */ namespace cryptonote { + enum class Transfer { + Normal, + Locked + }; + /*! * \brief Manages wallet operations. This is the most abstracted wallet class. */ @@ -157,7 +162,7 @@ namespace cryptonote bool show_incoming_transfers(const std::vector &args); bool show_payments(const std::vector &args); bool show_blockchain_height(const std::vector &args); - bool transfer_main(int transfer_type, const std::vector &args, bool called_by_mms); + bool transfer_main(Transfer transfer_type, const std::vector &args, bool called_by_mms); bool transfer(const std::vector &args); bool locked_transfer(const std::vector &args); bool stake(const std::vector &args_); @@ -168,8 +173,8 @@ namespace cryptonote bool locked_sweep_all(const std::vector &args); enum class sweep_type_t { stake, register_stake, all_or_below, single }; - bool sweep_main_internal(sweep_type_t sweep_type, std::vector &ptx_vector, cryptonote::address_parse_info const &dest); - bool sweep_main(uint64_t below, bool locked, const std::vector &args, tools::wallet2::sweep_style_t sweep_style = tools::wallet2::sweep_style_t::normal); + bool sweep_main_internal(sweep_type_t sweep_type, std::vector &ptx_vector, cryptonote::address_parse_info const &dest, bool blink); + bool sweep_main(uint64_t below, Transfer transfer_type, const std::vector &args); bool sweep_all(const std::vector &args); bool sweep_below(const std::vector &args); bool sweep_single(const std::vector &args); @@ -298,10 +303,12 @@ namespace cryptonote std::string get_mnemonic_language(); /*! - * \brief When --do-not-relay option is specified, save the raw tx hex blob to a file instead of calling m_wallet->commit_tx(ptx). + * \brief Submits or saves the transaction * \param ptx_vector Pending tx(es) created by transfer/sweep_all + * \param do_not_relay if true, save the raw tx hex blob to a file instead of calling m_wallet->commit_tx(ptx). + * \param blink true if this should be submitted as a blink tx */ - void commit_or_save(std::vector& ptx_vector, bool do_not_relay); + void commit_or_save(std::vector& ptx_vector, bool do_not_relay, bool blink); /*! * \brief checks whether background mining is enabled, and asks to configure it if not diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 9a8727da4..683a8ad1e 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -202,11 +202,16 @@ namespace return false; } - std::string get_text_reason(const cryptonote::COMMAND_RPC_SEND_RAW_TX::response &res, cryptonote::transaction const *tx) + std::string get_text_reason(const cryptonote::COMMAND_RPC_SEND_RAW_TX::response &res, cryptonote::transaction const *tx, bool blink) { + if (blink) { + return res.reason; + } + else { std::string reason = print_tx_verification_context (res.tvc, tx); reason += print_vote_verification_context(res.tvc.m_vote_ctx); return reason; + } } } @@ -6514,7 +6519,7 @@ crypto::hash wallet2::get_payment_id(const pending_tx &ptx) const } //---------------------------------------------------------------------------------------------------- // take a pending tx and actually send it to the daemon -void wallet2::commit_tx(pending_tx& ptx) +void wallet2::commit_tx(pending_tx& ptx, bool blink) { using namespace cryptonote; @@ -6525,6 +6530,7 @@ void wallet2::commit_tx(pending_tx& ptx) oreq.address = get_account().get_public_address_str(m_nettype); oreq.view_key = string_tools::pod_to_hex(get_account().get_keys().m_view_secret_key); oreq.tx = epee::string_tools::buff_to_hex_nodelimer(tx_to_blob(ptx.tx)); + oreq.blink = blink; m_daemon_rpc_mutex.lock(); bool r = invoke_http_json("/submit_raw_tx", oreq, ores, rpc_timeout, "POST"); m_daemon_rpc_mutex.unlock(); @@ -6539,13 +6545,14 @@ void wallet2::commit_tx(pending_tx& ptx) req.tx_as_hex = epee::string_tools::buff_to_hex_nodelimer(tx_to_blob(ptx.tx)); req.do_not_relay = false; req.do_sanity_checks = true; + req.blink = blink; COMMAND_RPC_SEND_RAW_TX::response daemon_send_resp; m_daemon_rpc_mutex.lock(); bool r = invoke_http_json("/sendrawtransaction", req, daemon_send_resp, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "sendrawtransaction"); THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "sendrawtransaction"); - THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status != CORE_RPC_STATUS_OK, error::tx_rejected, ptx.tx, get_rpc_status(daemon_send_resp.status), get_text_reason(daemon_send_resp, &ptx.tx)); + THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status != CORE_RPC_STATUS_OK, error::tx_rejected, ptx.tx, get_rpc_status(daemon_send_resp.status), get_text_reason(daemon_send_resp, &ptx.tx, blink)); // sanity checks for (size_t idx: ptx.selected_transfers) { @@ -6592,11 +6599,11 @@ void wallet2::commit_tx(pending_tx& ptx) << "Please, wait for confirmation for your balance to be unlocked."); } -void wallet2::commit_tx(std::vector& ptx_vector) +void wallet2::commit_tx(std::vector& ptx_vector, bool blink) { for (auto & ptx : ptx_vector) { - commit_tx(ptx); + commit_tx(ptx, blink); } } //---------------------------------------------------------------------------------------------------- @@ -7330,16 +7337,11 @@ bool wallet2::sign_multisig_tx_from_file(const std::string &filename, std::vecto //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm) const { - static const struct fee_multipliers_t - { - uint64_t values[4]; - } - multipliers[] = - { - { {1, 4, 20, 166} }, - { {1, 5, 25, 1000} }, - { {1, 5, 25, 125} }, - }; + static constexpr std::array, 3> multipliers = {{ + {{1, 4, 20, 166}}, + {{1, 5, 25, 1000}}, + {{1, 5, 25, 125}}, + }}; if (fee_algorithm == -1) fee_algorithm = get_fee_algorithm(); @@ -7348,21 +7350,19 @@ uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm) const if (priority == 0) priority = m_default_priority; if (priority == 0) + priority = fee_algorithm >= 1 ? 2 : 1; + + if (priority == BLINK_PRIORITY) { - if (fee_algorithm >= 1) - priority = 2; - else - priority = 1; + THROW_WALLET_EXCEPTION_IF(!use_fork_rules(HF_VERSION_BLINK, 0), error::invalid_priority); + return BLINK_TX_FEE_MULTIPLE; } - THROW_WALLET_EXCEPTION_IF(fee_algorithm < 0 || fee_algorithm >= (int)(loki::array_count(multipliers)), error::invalid_priority); - fee_multipliers_t const *curr_multiplier = multipliers + fee_algorithm; - if (priority >= 1 && priority <= (uint32_t)loki::array_count(curr_multiplier->values)) - { - return curr_multiplier->values[priority-1]; - } - - THROW_WALLET_EXCEPTION(error::invalid_priority); + THROW_WALLET_EXCEPTION_IF( + fee_algorithm < 0 || fee_algorithm >= (int)multipliers.size() || + priority < 1 || priority > multipliers[0].size(), + error::invalid_priority); + return multipliers[fee_algorithm][priority-1]; } //---------------------------------------------------------------------------------------------------- byte_and_output_fees wallet2::get_dynamic_base_fee_estimate() const @@ -7955,6 +7955,14 @@ wallet2::stake_result wallet2::create_stake_tx(const crypto::public_key& service try { priority = adjust_priority(priority); + + if (priority == BLINK_PRIORITY) + { + result.status = stake_result_status::no_blink; + result.msg += tr("Service node stakes cannot use blink priority"); + return result; + } + auto ptx_vector = create_transactions_2(dsts, CRYPTONOTE_DEFAULT_TX_MIXIN, unlock_at_block, priority, extra, subaddr_account, subaddr_indices, true); if (ptx_vector.size() == 1) { @@ -8007,6 +8015,14 @@ wallet2::register_service_node_result wallet2::create_register_service_node_tx(c local_args.erase(local_args.begin()); priority = adjust_priority(priority); + + if (priority == BLINK_PRIORITY) + { + result.status = register_service_node_result_status::no_blink; + result.msg += tr("Service node registrations cannot use blink priority"); + return result; + } + if (local_args.size() < 6) { result.status = register_service_node_result_status::insufficient_num_args; @@ -14021,7 +14037,7 @@ uint64_t wallet2::hash_m_transfers(int64_t transfer_height, crypto::hash &hash) return current_height; } -const std::array allowed_priority_strings = {{"default", "unimportant", "normal", "elevated", "priority"}}; +constexpr std::array allowed_priority_strings = {{"default", "unimportant", "normal", "elevated", "priority"}}; bool parse_subaddress_indices(const std::string& arg, std::set& subaddr_indices, std::string *err_msg) { subaddr_indices.clear(); @@ -14055,6 +14071,9 @@ bool parse_priority(const std::string& arg, uint32_t& priority) if(priority_pos != allowed_priority_strings.end()) { priority = std::distance(allowed_priority_strings.begin(), priority_pos); return true; + } else if (arg == "blink") { + priority = wallet2::BLINK_PRIORITY; + return true; } return false; } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index f3c59bd2a..d8ddb1d36 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -307,6 +307,7 @@ private: friend class wallet_keys_unlocker; friend class wallet_device_callback; public: + static constexpr uint32_t BLINK_PRIORITY = 0x626c6e6b; // "blnk" static constexpr const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::chrono::seconds(30); enum RefreshType { @@ -923,8 +924,8 @@ private: std::vector> &outs, uint64_t unlock_time, uint64_t fee, const std::vector& extra, cryptonote::transaction& tx, pending_tx &ptx, const rct::RCTConfig &rct_config, bool is_staking_tx = false); - void commit_tx(pending_tx& ptx_vector); - void commit_tx(std::vector& ptx_vector); + void commit_tx(pending_tx& ptx_vector, bool blink = false); + void commit_tx(std::vector& ptx_vector, bool blink = false); bool save_tx(const std::vector& ptx_vector, const std::string &filename) const; std::string dump_tx_to_str(const std::vector &ptx_vector) const; std::string save_multisig_tx(multisig_tx_set txs); @@ -1447,6 +1448,7 @@ private: service_node_contributors_maxed, service_node_insufficient_contribution, too_many_transactions_constructed, + no_blink, }; struct stake_result @@ -1483,6 +1485,7 @@ private: wallet_not_synced, too_many_transactions_constructed, exception_thrown, + no_blink, }; struct register_service_node_result