oxen-io/oxen-core
2
1
Fork 0
mirror of https://github.com/oxen-io/oxen-core.git synced 2023-12-14 02:22:56 +01:00
Code Issues Projects Releases Wiki Activity

run clang format

Browse source
This commit is contained in:
Jason Rhinelander 2023-04-13 10:50:13 -03:00
parent 45db02de78
commit 13409ad00e
No known key found for this signature in database
GPG key ID: C4992CE7A88D4262
581 changed files with 185777 additions and 157791 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
pybind/common.hpp
View file

@ -1,34 +1,25 @@
#pragma once
#include <pybind11/functional.h>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/functional.h>
namespace py = pybind11;
namespace wallet
{
void
Wallet_Init(py::module& mod);
namespace wallet {
void Wallet_Init(py::module& mod);
void
Keyring_Init(py::module& mod);
void Keyring_Init(py::module& mod);
void
KeyringManager_Init(py::module& mod);
void KeyringManager_Init(py::module& mod);
void
WalletConfig_Init(py::module& mod);
void WalletConfig_Init(py::module& mod);
void
GeneralWalletConfig_Init(py::module& mod);
void GeneralWalletConfig_Init(py::module& mod);
void
DaemonCommsConfig_Init(py::module& mod);
void DaemonCommsConfig_Init(py::module& mod);
void
RPCConfig_Init(py::module& mod);
void RPCConfig_Init(py::module& mod);
void
LoggingConfig_Init(py::module& mod);
void LoggingConfig_Init(py::module& mod);
} // namespace wallet

3
pybind/module.cpp
View file

@ -1,7 +1,6 @@
#include "common.hpp"
PYBIND11_MODULE(pywallet3, m)
{
PYBIND11_MODULE(pywallet3, m) {
wallet::Wallet_Init(m);
wallet::Keyring_Init(m);
wallet::KeyringManager_Init(m);

7
pybind/wallet/daemon_comms_config.cpp
View file

@ -1,11 +1,8 @@
#include "../common.hpp"
#include "wallet3/config/config.hpp"
namespace wallet
{
void
DaemonCommsConfig_Init(py::module& mod)
{
namespace wallet {
void DaemonCommsConfig_Init(py::module& mod) {
py::class_<DaemonCommsConfig, std::shared_ptr<DaemonCommsConfig>>(mod, "DaemonCommsConfig")
.def(py::init<>())
.def_readwrite("address", &DaemonCommsConfig::address);

14
pybind/wallet/general_wallet_config.cpp
View file

@ -1,15 +1,15 @@
#include "../common.hpp"
#include "wallet3/config/config.hpp"
namespace wallet
{
void
GeneralWalletConfig_Init(py::module& mod)
{
py::class_<GeneralWalletConfig, std::shared_ptr<GeneralWalletConfig>>(mod, "GeneralWalletConfig")
namespace wallet {
void GeneralWalletConfig_Init(py::module& mod) {
py::class_<GeneralWalletConfig, std::shared_ptr<GeneralWalletConfig>>(
mod, "GeneralWalletConfig")
.def(py::init<>())
.def_readwrite("datadir", &GeneralWalletConfig::datadir)
.def_readwrite("append_network_type_to_datadir", &GeneralWalletConfig::append_network_type_to_datadir);
.def_readwrite(
"append_network_type_to_datadir",
&GeneralWalletConfig::append_network_type_to_datadir);
}
} // namespace wallet

27
pybind/wallet/keyring.cpp
View file

@ -1,20 +1,24 @@
#include "../common.hpp"
#include "wallet3/keyring.hpp"
#include <crypto/crypto.h>
#include <common/hex.h>
#include <crypto/crypto.h>
#include <cryptonote_basic/cryptonote_basic.h>
namespace wallet
{
void
Keyring_Init(py::module& mod)
{
#include "../common.hpp"
namespace wallet {
void Keyring_Init(py::module& mod) {
py::class_<Keyring, std::shared_ptr<Keyring>>(mod, "Keyring")
.def(py::init([]( std::string ssk, std::string spk, std::string vsk, std::string vpk, std::string nettype) {
.def(py::init([](std::string ssk,
std::string spk,
std::string vsk,
std::string vpk,
std::string nettype) {
auto type = cryptonote::network_type::MAINNET;
if (nettype == "testnet") type = cryptonote::network_type::TESTNET;
else if (nettype == "devnet") type = cryptonote::network_type::DEVNET;
if (nettype == "testnet")
type = cryptonote::network_type::TESTNET;
else if (nettype == "devnet")
type = cryptonote::network_type::DEVNET;
crypto::secret_key spend_priv;
crypto::public_key spend_pub;
crypto::secret_key view_priv;
@ -23,7 +27,8 @@ namespace wallet
tools::hex_to_type<crypto::public_key>(spk, spend_pub);
tools::hex_to_type<crypto::secret_key>(vsk, view_priv);
tools::hex_to_type<crypto::public_key>(vpk, view_pub);
return Keyring(spend_priv, spend_pub, view_priv, view_pub, std::move(type)); }))
return Keyring(spend_priv, spend_pub, view_priv, view_pub, std::move(type));
}))
.def("get_main_address", &Keyring::get_main_address);
}

22
pybind/wallet/keyring_manager.cpp
View file

@ -1,22 +1,24 @@
#include "../common.hpp"
#include "wallet3/keyring_manager.hpp"
#include <crypto/crypto.h>
#include <cryptonote_basic/cryptonote_basic.h>
namespace wallet
{
void
KeyringManager_Init(py::module& mod)
{
#include "../common.hpp"
namespace wallet {
void KeyringManager_Init(py::module& mod) {
py::class_<KeyringManager, std::shared_ptr<KeyringManager>>(mod, "KeyringManager")
.def(py::init([](std::string nettype) {
auto type = cryptonote::network_type::MAINNET;
if (nettype == "testnet") type = cryptonote::network_type::TESTNET;
else if (nettype == "devnet") type = cryptonote::network_type::DEVNET;
return KeyringManager(std::move(type)); }))
if (nettype == "testnet")
type = cryptonote::network_type::TESTNET;
else if (nettype == "devnet")
type = cryptonote::network_type::DEVNET;
return KeyringManager(std::move(type));
}))
.def("generate_keyring_from_electrum_seed", &KeyringManager::generate_keyring_from_electrum_seed);
.def("generate_keyring_from_electrum_seed",
&KeyringManager::generate_keyring_from_electrum_seed);
}
} // namespace wallet

7
pybind/wallet/logging_config.cpp
View file

@ -1,11 +1,8 @@
#include "../common.hpp"
#include "wallet3/config/config.hpp"
namespace wallet
{
void
LoggingConfig_Init(py::module& mod)
{
namespace wallet {
void LoggingConfig_Init(py::module& mod) {
py::class_<LoggingConfig, std::shared_ptr<LoggingConfig>>(mod, "LoggingConfig")
.def(py::init<>())
.def_readwrite("level", &LoggingConfig::level)

7
pybind/wallet/rpc_config.cpp
View file

@ -1,11 +1,8 @@
#include "../common.hpp"
#include "wallet3/config/config.hpp"
namespace wallet
{
void
RPCConfig_Init(py::module& mod)
{
namespace wallet {
void RPCConfig_Init(py::module& mod) {
py::class_<rpc::Config, std::shared_ptr<rpc::Config>>(mod, "RPCConfig")
.def(py::init<>())
.def_readwrite("sockname", &rpc::Config::sockname);

35
pybind/wallet/wallet.cpp
View file

@ -1,18 +1,21 @@
#include "../common.hpp"
#include <oxenmq/oxenmq.h>
#include <wallet3/wallet.hpp>
#include <oxen/log.hpp>
#include <wallet3/config/config.hpp>
#include <wallet3/default_daemon_comms.hpp>
#include <wallet3/keyring.hpp>
#include <wallet3/config/config.hpp>
#include <oxenmq/oxenmq.h>
#include <oxen/log.hpp>
#include <wallet3/wallet.hpp>
#include "../common.hpp"
static auto logcat = oxen::log::Cat("omq");
void omq_logger(oxenmq::LogLevel level, const char* file, int line, std::string message) {
constexpr std::string_view format = "[{}:{}]: {}";
switch (level) {
case oxenmq::LogLevel::fatal: oxen::log::critical(logcat, format, file, line, message); break;
case oxenmq::LogLevel::fatal:
oxen::log::critical(logcat, format, file, line, message);
break;
case oxenmq::LogLevel::error: oxen::log::error(logcat, format, file, line, message); break;
case oxenmq::LogLevel::warn: oxen::log::warning(logcat, format, file, line, message); break;
case oxenmq::LogLevel::info: oxen::log::info(logcat, format, file, line, message); break;
@ -21,18 +24,24 @@ void omq_logger(oxenmq::LogLevel level, const char* file, int line, std::string
}
}
namespace wallet
{
void
Wallet_Init(py::module& mod)
{
namespace wallet {
void Wallet_Init(py::module& mod) {
py::class_<Wallet, std::shared_ptr<Wallet>>(mod, "Wallet")
.def(py::init([](const std::string& wallet_name, std::shared_ptr<Keyring> keyring, Config config) {
.def(py::init([](const std::string& wallet_name,
std::shared_ptr<Keyring> keyring,
Config config) {
auto& comms_config = config.daemon;
auto& omq_rpc_config = config.omq_rpc;
auto oxenmq = std::make_shared<oxenmq::OxenMQ>(omq_logger, oxenmq::LogLevel::info);
auto comms = std::make_shared<DefaultDaemonComms>(oxenmq, comms_config);
return Wallet::create(std::move(oxenmq), std::move(keyring), nullptr, std::move(comms), wallet_name + ".sqlite", "", std::move(config));
return Wallet::create(
std::move(oxenmq),
std::move(keyring),
nullptr,
std::move(comms),
wallet_name + ".sqlite",
"",
std::move(config));
}))
.def("get_balance", &Wallet::get_balance)
.def("deregister", &Wallet::deregister);

7
pybind/wallet/wallet_config.cpp
View file

@ -1,11 +1,8 @@
#include "../common.hpp"
#include "wallet3/config/config.hpp"
namespace wallet
{
void
WalletConfig_Init(py::module& mod)
{
namespace wallet {
void WalletConfig_Init(py::module& mod) {
py::class_<Config, std::shared_ptr<Config>>(mod, "WalletConfig")
.def(py::init<>())
.def_readwrite("general", &Config::general)

282
src/blockchain_db/blockchain_db.cpp
View file

@ -27,73 +27,66 @@
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "common/string_util.h"
#include "cryptonote_basic/hardfork.h"
#include "cryptonote_core/service_node_rules.h"
#include "checkpoints/checkpoints.h"
#include "epee/string_tools.h"
#include "blockchain_db.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "ringct/rctOps.h"
#include "common/hex.h"
#include "lmdb/db_lmdb.h"
#include <chrono>
namespace cryptonote
{
#include "checkpoints/checkpoints.h"
#include "common/hex.h"
#include "common/string_util.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "cryptonote_basic/hardfork.h"
#include "cryptonote_core/service_node_rules.h"
#include "epee/string_tools.h"
#include "lmdb/db_lmdb.h"
#include "ringct/rctOps.h"
namespace cryptonote {
static auto logcat = log::Cat("blockchain.db");
const command_line::arg_descriptor<std::string> arg_db_sync_mode = {
"db-sync-mode"
, "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[<nblocks_per_sync>[blocks]|<nbytes_per_sync>[bytes]]."
, "fast:async:250000000bytes"
};
"db-sync-mode",
"Specify sync option, using format "
"[safe|fast|fastest]:[sync|async]:[<nblocks_per_sync>[blocks]|<nbytes_per_sync>[bytes]].",
"fast:async:250000000bytes"};
const command_line::arg_descriptor<bool> arg_db_salvage = {
"db-salvage"
, "Try to salvage a blockchain database if it seems corrupted"
, false
};
"db-salvage", "Try to salvage a blockchain database if it seems corrupted", false};
BlockchainDB *new_db()
{
BlockchainDB* new_db() {
return new BlockchainLMDB();
}
void BlockchainDB::init_options(boost::program_options::options_description& desc)
{
void BlockchainDB::init_options(boost::program_options::options_description& desc) {
command_line::add_arg(desc, arg_db_sync_mode);
command_line::add_arg(desc, arg_db_salvage);
}
void BlockchainDB::pop_block()
{
void BlockchainDB::pop_block() {
block blk;
std::vector<transaction> txs;
pop_block(blk, txs);
}
void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, std::string>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr)
{
void BlockchainDB::add_transaction(
const crypto::hash& blk_hash,
const std::pair<transaction, std::string>& txp,
const crypto::hash* tx_hash_ptr,
const crypto::hash* tx_prunable_hash_ptr) {
const transaction& tx = txp.first;
bool miner_tx = false;
crypto::hash tx_hash, tx_prunable_hash;
if (!tx_hash_ptr)
{
if (!tx_hash_ptr) {
// should only need to compute hash for miner transactions
tx_hash = get_transaction_hash(tx);
log::trace(logcat, "null tx_hash_ptr - needed to compute: {}", tx_hash);
}
else
{
} else {
tx_hash = *tx_hash_ptr;
}
bool has_blacklisted_outputs = false;
if (tx.version >= cryptonote::txversion::v2_ringct)
{
if (tx.version >= cryptonote::txversion::v2_ringct) {
if (!tx_prunable_hash_ptr)
tx_prunable_hash = get_transaction_prunable_hash(tx, &txp.second);
else
@ -101,28 +94,24 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair
crypto::secret_key secret_tx_key;
cryptonote::account_public_address address;
if (get_tx_secret_key_from_tx_extra(tx.extra, secret_tx_key) && get_service_node_contributor_from_tx_extra(tx.extra, address))
if (get_tx_secret_key_from_tx_extra(tx.extra, secret_tx_key) &&
get_service_node_contributor_from_tx_extra(tx.extra, address))
has_blacklisted_outputs = true;
}
for (const txin_v& tx_input : tx.vin)
{
if (std::holds_alternative<txin_to_key>(tx_input))
{
for (const txin_v& tx_input : tx.vin) {
if (std::holds_alternative<txin_to_key>(tx_input)) {
add_spent_key(var::get<txin_to_key>(tx_input).k_image);
}
else if (std::holds_alternative<txin_gen>(tx_input))
{
} else if (std::holds_alternative<txin_gen>(tx_input)) {
/* nothing to do here */
miner_tx = true;
}
else
{
log::info(logcat, "Unsupported input type, removing key images and aborting transaction addition");
for (const txin_v& tx_input : tx.vin)
{
if (std::holds_alternative<txin_to_key>(tx_input))
{
} else {
log::info(
logcat,
"Unsupported input type, removing key images and aborting transaction "
"addition");
for (const txin_v& tx_input : tx.vin) {
if (std::holds_alternative<txin_to_key>(tx_input)) {
remove_spent_key(var::get<txin_to_key>(tx_input).k_image);
}
}
@ -136,32 +125,30 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair
// iterate tx.vout using indices instead of C++11 foreach syntax because
// we need the index
for (uint64_t i = 0; i < tx.vout.size(); ++i)
{
for (uint64_t i = 0; i < tx.vout.size(); ++i) {
uint64_t unlock_time = 0;
if (tx.version >= cryptonote::txversion::v3_per_output_unlock_times)
{
if (tx.version >= cryptonote::txversion::v3_per_output_unlock_times) {
unlock_time = tx.output_unlock_times[i];
}
else
{
} else {
unlock_time = tx.unlock_time;
}
// miner v2 txes have their coinbase output in one single out to save space,
// and we store them as rct outputs with an identity mask
if (miner_tx && tx.version >= cryptonote::txversion::v2_ringct)
{
if (miner_tx && tx.version >= cryptonote::txversion::v2_ringct) {
cryptonote::tx_out vout = tx.vout[i];
const rct::key commitment = rct::zeroCommit(vout.amount);
vout.amount = 0;
amount_output_indices[i] = add_output(tx_hash, vout, i, unlock_time,
&commitment);
}
else
{
amount_output_indices[i] = add_output(tx_hash, tx.vout[i], i, unlock_time,
tx.version >= cryptonote::txversion::v2_ringct ? &tx.rct_signatures.outPk[i].mask : NULL);
amount_output_indices[i] = add_output(tx_hash, vout, i, unlock_time, &commitment);
} else {
amount_output_indices[i] = add_output(
tx_hash,
tx.vout[i],
i,
unlock_time,
tx.version >= cryptonote::txversion::v2_ringct
? &tx.rct_signatures.outPk[i].mask
: NULL);
}
}
@ -171,14 +158,13 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair
add_tx_amount_output_indices(tx_id, amount_output_indices);
}
uint64_t BlockchainDB::add_block( const std::pair<block, std::string>& blck
, size_t block_weight
, uint64_t long_term_block_weight
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, const std::vector<std::pair<transaction, std::string>>& txs
)
{
uint64_t BlockchainDB::add_block(
const std::pair<block, std::string>& blck,
size_t block_weight,
uint64_t long_term_block_weight,
const difficulty_type& cumulative_difficulty,
const uint64_t& coins_generated,
const std::vector<std::pair<transaction, std::string>>& txs) {
const block& blk = blck.first;
// sanity
@ -202,12 +188,10 @@ uint64_t BlockchainDB::add_block( const std::pair<block, std::string>& blck
int tx_i = 0;
crypto::hash tx_hash{};
for (const std::pair<transaction, std::string>& tx : txs)
{
for (const std::pair<transaction, std::string>& tx : txs) {
tx_hash = blk.tx_hashes[tx_i];
add_transaction(blk_hash, tx, &tx_hash);
for (const auto &vout: tx.first.vout)
{
for (const auto& vout : tx.first.vout) {
if (vout.amount == 0)
++num_rct_outs;
}
@ -217,7 +201,14 @@ uint64_t BlockchainDB::add_block( const std::pair<block, std::string>& blck
// call out to subclass implementation to add the block & metadata
started = std::chrono::steady_clock::now();
add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash);
add_block(
blk,
block_weight,
long_term_block_weight,
cumulative_difficulty,
coins_generated,
num_rct_outs,
blk_hash);
time_add_block1 += std::chrono::steady_clock::now() - started;
++num_calls;
@ -225,14 +216,12 @@ uint64_t BlockchainDB::add_block( const std::pair<block, std::string>& blck
return prev_height;
}
void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs)
{
void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs) {
blk = get_top_block();
remove_block();
for (auto it = blk.tx_hashes.rbegin(); it != blk.tx_hashes.rend(); ++it)
{
for (auto it = blk.tx_hashes.rbegin(); it != blk.tx_hashes.rend(); ++it) {
auto& h = *it;
cryptonote::transaction tx;
if (!get_tx(h, tx) && !get_pruned_tx(h, tx))
@ -243,14 +232,11 @@ void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs)
remove_transaction(get_transaction_hash(blk.miner_tx));
}
void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
{
void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) {
transaction tx = get_pruned_tx(tx_hash);
for (const txin_v& tx_input : tx.vin)
{
if (std::holds_alternative<txin_to_key>(tx_input))
{
for (const txin_v& tx_input : tx.vin) {
if (std::holds_alternative<txin_to_key>(tx_input)) {
remove_spent_key(var::get<txin_to_key>(tx_input).k_image);
}
}
@ -259,20 +245,17 @@ void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
remove_transaction_data(tx_hash, tx);
}
block_header BlockchainDB::get_block_header(const crypto::hash& h) const
{
block_header BlockchainDB::get_block_header(const crypto::hash& h) const {
block_header b = get_block_header_from_height(get_block_height(h));
return b;
}
block BlockchainDB::get_block(const crypto::hash& h) const
{
block BlockchainDB::get_block(const crypto::hash& h) const {
block b = get_block_from_height(get_block_height(h));
return b;
}
bool BlockchainDB::get_tx(const crypto::hash& h, cryptonote::transaction &tx) const
{
bool BlockchainDB::get_tx(const crypto::hash& h, cryptonote::transaction& tx) const {
std::string bd;
if (!get_tx_blob(h, bd))
return false;
@ -282,43 +265,38 @@ bool BlockchainDB::get_tx(const crypto::hash& h, cryptonote::transaction &tx) co
return true;
}
bool BlockchainDB::get_pruned_tx(const crypto::hash& h, cryptonote::transaction &tx) const
{
bool BlockchainDB::get_pruned_tx(const crypto::hash& h, cryptonote::transaction& tx) const {
std::string bd;
if (!get_pruned_tx_blob(h, bd))
return false;
if (!parse_and_validate_tx_base_from_blob(bd, tx))
{
if (!parse_and_validate_tx_base_from_blob(bd, tx)) {
throw DB_ERROR("Failed to parse transaction base from blob retrieved from the db");
}
return true;
}
transaction BlockchainDB::get_tx(const crypto::hash& h) const
{
transaction BlockchainDB::get_tx(const crypto::hash& h) const {
transaction tx;
if (!get_tx(h, tx))
throw TX_DNE("tx with hash " + tools::type_to_hex(h) + " not found in db");
return tx;
}
uint64_t BlockchainDB::get_output_unlock_time(const uint64_t amount, const uint64_t amount_index) const
{
uint64_t BlockchainDB::get_output_unlock_time(
const uint64_t amount, const uint64_t amount_index) const {
output_data_t odata = get_output_key(amount, amount_index);
return odata.unlock_time;
}
transaction BlockchainDB::get_pruned_tx(const crypto::hash& h) const
{
transaction BlockchainDB::get_pruned_tx(const crypto::hash& h) const {
transaction tx;
if (!get_pruned_tx(h, tx))
throw TX_DNE("pruned tx with hash " + tools::type_to_hex(h) + " not found in db");
return tx;
}
void BlockchainDB::reset_stats()
{
void BlockchainDB::reset_stats() {
num_calls = 0;
time_blk_hash = 0ns;
time_tx_exists = 0ns;
@ -327,9 +305,10 @@ void BlockchainDB::reset_stats()
time_commit1 = 0ns;
}
void BlockchainDB::show_stats()
{
log::info(logcat, "\n*********************************\n \
void BlockchainDB::show_stats() {
log::info(
logcat,
"\n*********************************\n \
num_calls: {}\n \
time_blk_hash: {}\n \
time_tx_exists: {}\n \
@ -337,11 +316,15 @@ void BlockchainDB::show_stats()
time_add_transaction: {}\n \
time_commit1: {}\n \
*********************************\n",
num_calls, tools::friendly_duration(time_blk_hash), tools::friendly_duration(time_tx_exists), tools::friendly_duration(time_add_block1), tools::friendly_duration(time_add_transaction), tools::friendly_duration(time_commit1));
num_calls,
tools::friendly_duration(time_blk_hash),
tools::friendly_duration(time_tx_exists),
tools::friendly_duration(time_add_block1),
tools::friendly_duration(time_add_transaction),
tools::friendly_duration(time_commit1));
}
void BlockchainDB::fixup(cryptonote::network_type)
{
void BlockchainDB::fixup(cryptonote::network_type) {
if (is_read_only()) {
log::info(logcat, "Database is opened read only - skipping fixup check");
return;
@ -350,11 +333,13 @@ void BlockchainDB::fixup(cryptonote::network_type)
set_batch_transactions(true);
}
bool BlockchainDB::get_immutable_checkpoint(checkpoint_t *immutable_checkpoint, uint64_t block_height) const
{
bool BlockchainDB::get_immutable_checkpoint(
checkpoint_t* immutable_checkpoint, uint64_t block_height) const {
size_t constexpr NUM_CHECKPOINTS = service_nodes::CHECKPOINT_NUM_CHECKPOINTS_FOR_CHAIN_FINALITY;
static_assert(NUM_CHECKPOINTS == 2,
"Expect checkpoint finality to be 2, otherwise the immutable logic needs to check for any hardcoded "
static_assert(
NUM_CHECKPOINTS == 2,
"Expect checkpoint finality to be 2, otherwise the immutable logic needs to check for "
"any hardcoded "
"checkpoints inbetween");
std::vector<checkpoint_t> checkpoints = get_checkpoints_range(block_height, 0, NUM_CHECKPOINTS);
@ -363,20 +348,18 @@ bool BlockchainDB::get_immutable_checkpoint(checkpoint_t *immutable_checkpoint,
return false;
checkpoint_t* checkpoint_ptr = nullptr;
if (checkpoints[0].type != checkpoint_type::service_node) // checkpoint[0] is the first closest checkpoint that is <= my height
if (checkpoints[0].type != checkpoint_type::service_node) // checkpoint[0] is the first closest
// checkpoint that is <= my height
{
checkpoint_ptr = &checkpoints[0]; // Must be hard-coded then, always immutable
}
else if (checkpoints.size() == NUM_CHECKPOINTS)
{
} else if (checkpoints.size() == NUM_CHECKPOINTS) {
// NOTE: The first checkpoint is a service node checkpoint. Go back
// 1 checkpoint, which will either be another service node checkpoint or
// a predefined one.
checkpoint_ptr = &checkpoints[1];
}
else
{
return false; // NOTE: Only one service node checkpoint recorded, we can override this checkpoint.
} else {
return false; // NOTE: Only one service node checkpoint recorded, we can override this
// checkpoint.
}
if (immutable_checkpoint)
@ -385,11 +368,9 @@ bool BlockchainDB::get_immutable_checkpoint(checkpoint_t *immutable_checkpoint,
return true;
}
uint64_t BlockchainDB::get_tx_block_height(const crypto::hash &h) const
{
uint64_t BlockchainDB::get_tx_block_height(const crypto::hash& h) const {
auto result = get_tx_block_heights({{h}}).front();
if (result == std::numeric_limits<uint64_t>::max())
{
if (result == std::numeric_limits<uint64_t>::max()) {
std::string err = "tx_data_t with hash " + tools::type_to_hex(h) + " not found in db";
log::info(logcat, "{}", err);
throw TX_DNE(std::move(err));
@ -397,34 +378,33 @@ uint64_t BlockchainDB::get_tx_block_height(const crypto::hash &h) const
return result;
}
bool BlockchainDB::get_alt_block_header(const crypto::hash &blkid, alt_block_data_t *data, cryptonote::block_header *header, std::string *checkpoint) const
{
bool BlockchainDB::get_alt_block_header(
const crypto::hash& blkid,
alt_block_data_t* data,
cryptonote::block_header* header,
std::string* checkpoint) const {
std::string blob;
if (!get_alt_block(blkid, data, &blob, checkpoint))
{
if (!get_alt_block(blkid, data, &blob, checkpoint)) {
throw BLOCK_DNE("Alt-block with hash " + tools::type_to_hex(blkid) + " not found in db");
return false;
}
try
{
try {
serialization::binary_string_unarchiver ba{blob};
serialization::value(ba, *header);
}
catch(std::exception &e)
{
} catch (std::exception& e) {
return false;
}
return true;
}
void BlockchainDB::fill_timestamps_and_difficulties_for_pow(cryptonote::network_type nettype,
void BlockchainDB::fill_timestamps_and_difficulties_for_pow(
cryptonote::network_type nettype,
std::vector<uint64_t>& timestamps,
std::vector<uint64_t>& difficulties,
uint64_t chain_height,
uint64_t timestamps_difficulty_height) const
{
uint64_t timestamps_difficulty_height) const {
constexpr uint64_t MIN_CHAIN_HEIGHT = 2;
if (chain_height < MIN_CHAIN_HEIGHT)
return;
@ -436,35 +416,33 @@ void BlockchainDB::fill_timestamps_and_difficulties_for_pow(cryptonote::network_
timestamps.reserve(block_count);
difficulties.reserve(block_count);
if (timestamps_difficulty_height == 0 ||
(chain_height - timestamps_difficulty_height) != 1 ||
timestamps.size() > block_count ||
difficulties.size() > block_count)
{
if (timestamps_difficulty_height == 0 || (chain_height - timestamps_difficulty_height) != 1 ||
timestamps.size() > block_count || difficulties.size() > block_count) {
// Cache invalidated.
timestamps.clear();
difficulties.clear();
// Fill missing timestamps/difficulties, up to one before the latest (latest is added below).
// Fill missing timestamps/difficulties, up to one before the latest (latest is added
// below).
uint64_t start_height = chain_height - std::min<size_t>(chain_height, block_count);
start_height = std::max<uint64_t>(start_height, 1);
for (uint64_t block_height = start_height; block_height < (chain_height - 1) /*skip latest block*/; block_height++)
{
for (uint64_t block_height = start_height;
block_height < (chain_height - 1) /*skip latest block*/;
block_height++) {
timestamps.push_back(get_block_timestamp(block_height));
difficulties.push_back(get_block_cumulative_difficulty(block_height));
}
}
// Add latest timestamp/difficulty
add_timestamp_and_difficulty(nettype,
add_timestamp_and_difficulty(
nettype,
chain_height,
timestamps,
difficulties,
get_block_timestamp(top_block_height),
get_block_cumulative_difficulty(top_block_height));
}
} // namespace cryptonote

299
src/blockchain_db/blockchain_db.h
View file

@ -30,9 +30,10 @@
#pragma once
#include <string>
#include <exception>
#include <boost/program_options.hpp>
#include <exception>
#include <string>
#include "common/command_line.h"
#include "common/fs.h"
#include "crypto/hash.h"
@ -99,8 +100,7 @@ namespace service_nodes {
struct proof_info;
}
namespace cryptonote
{
namespace cryptonote {
struct checkpoint_t;
/** a pair of <transaction hash, output index>, typedef for convenience */
@ -114,8 +114,7 @@ extern const command_line::arg_descriptor<bool, false> arg_db_salvage;
/**
* @brief a struct containing output metadata
*/
struct output_data_t
{
struct output_data_t {
crypto::public_key pubkey; //!< the output's public key (for spend verification)
uint64_t unlock_time; //!< the output's unlock time (or height)
uint64_t height; //!< the height of the block which created the output
@ -124,24 +123,21 @@ struct output_data_t
#pragma pack(pop)
#pragma pack(push, 1)
struct tx_data_t
{
struct tx_data_t {
uint64_t tx_id;
uint64_t unlock_time;
uint64_t block_id;
};
#pragma pack(pop)
struct alt_block_data_1_t
{
struct alt_block_data_1_t {
uint64_t height;
uint64_t cumulative_weight;
uint64_t cumulative_difficulty;
uint64_t already_generated_coins;
};
struct alt_block_data_t
{
struct alt_block_data_t {
uint64_t height;
uint64_t cumulative_weight;
uint64_t cumulative_difficulty;
@ -152,8 +148,7 @@ struct alt_block_data_t
/**
* @brief a struct containing txpool per transaction metadata
*/
struct txpool_tx_meta_t
{
struct txpool_tx_meta_t {
crypto::hash max_used_block_id;
crypto::hash last_failed_id;
uint64_t weight;
@ -185,8 +180,7 @@ struct txpool_tx_meta_t
/**
* @brief A base class for BlockchainDB exceptions
*/
class DB_EXCEPTION : public std::runtime_error
{
class DB_EXCEPTION : public std::runtime_error {
public:
using std::runtime_error::runtime_error;
};
@ -194,8 +188,7 @@ class DB_EXCEPTION : public std::runtime_error
/**
* @brief A generic BlockchainDB exception
*/
class DB_ERROR : public DB_EXCEPTION
{
class DB_ERROR : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
DB_ERROR() : DB_EXCEPTION("Generic DB Error") {}
@ -204,8 +197,7 @@ class DB_ERROR : public DB_EXCEPTION
/**
* @brief thrown when there is an error starting a DB transaction
*/
class DB_ERROR_TXN_START : public DB_EXCEPTION
{
class DB_ERROR_TXN_START : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
DB_ERROR_TXN_START() : DB_EXCEPTION("DB Error in starting txn") {}
@ -214,8 +206,7 @@ class DB_ERROR_TXN_START : public DB_EXCEPTION
/**
* @brief thrown when opening the BlockchainDB fails
*/
class DB_OPEN_FAILURE : public DB_EXCEPTION
{
class DB_OPEN_FAILURE : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
DB_OPEN_FAILURE() : DB_EXCEPTION("Failed to open the db") {}
@ -224,8 +215,7 @@ class DB_OPEN_FAILURE : public DB_EXCEPTION
/**
* @brief thrown when creating the BlockchainDB fails
*/
class DB_CREATE_FAILURE : public DB_EXCEPTION
{
class DB_CREATE_FAILURE : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
DB_CREATE_FAILURE() : DB_EXCEPTION("Failed to create the db") {}
@ -234,8 +224,7 @@ class DB_CREATE_FAILURE : public DB_EXCEPTION
/**
* @brief thrown when synchronizing the BlockchainDB to disk fails
*/
class DB_SYNC_FAILURE : public DB_EXCEPTION
{
class DB_SYNC_FAILURE : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
DB_SYNC_FAILURE() : DB_EXCEPTION("Failed to sync the db") {}
@ -244,8 +233,7 @@ class DB_SYNC_FAILURE : public DB_EXCEPTION
/**
* @brief thrown when a requested block does not exist
*/
class BLOCK_DNE : public DB_EXCEPTION
{
class BLOCK_DNE : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
BLOCK_DNE() : DB_EXCEPTION("The block requested does not exist") {}
@ -254,8 +242,7 @@ class BLOCK_DNE : public DB_EXCEPTION
/**
* @brief thrown when a block's parent does not exist (and it needed to)
*/
class BLOCK_PARENT_DNE : public DB_EXCEPTION
{
class BLOCK_PARENT_DNE : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
BLOCK_PARENT_DNE() : DB_EXCEPTION("The parent of the block does not exist") {}
@ -264,8 +251,7 @@ class BLOCK_PARENT_DNE : public DB_EXCEPTION
/**
* @brief thrown when a block exists, but shouldn't, namely when adding a block
*/
class BLOCK_EXISTS : public DB_EXCEPTION
{
class BLOCK_EXISTS : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
BLOCK_EXISTS() : DB_EXCEPTION("The block to be added already exists!") {}
@ -274,8 +260,7 @@ class BLOCK_EXISTS : public DB_EXCEPTION
/**
* @brief thrown when something is wrong with the block to be added
*/
class BLOCK_INVALID : public DB_EXCEPTION
{
class BLOCK_INVALID : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
BLOCK_INVALID() : DB_EXCEPTION("The block to be added did not pass validation!") {}
@ -284,8 +269,7 @@ class BLOCK_INVALID : public DB_EXCEPTION
/**
* @brief thrown when a requested transaction does not exist
*/
class TX_DNE : public DB_EXCEPTION
{
class TX_DNE : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
TX_DNE() : DB_EXCEPTION("The transaction requested does not exist") {}
@ -294,8 +278,7 @@ class TX_DNE : public DB_EXCEPTION
/**
* @brief thrown when a transaction exists, but shouldn't, namely when adding a block
*/
class TX_EXISTS : public DB_EXCEPTION
{
class TX_EXISTS : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
TX_EXISTS() : DB_EXCEPTION("The transaction to be added already exists!") {}
@ -304,8 +287,7 @@ class TX_EXISTS : public DB_EXCEPTION
/**
* @brief thrown when a requested output does not exist
*/
class OUTPUT_DNE : public DB_EXCEPTION
{
class OUTPUT_DNE : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
OUTPUT_DNE() : DB_EXCEPTION("The output requested does not exist!") {}
@ -314,8 +296,7 @@ class OUTPUT_DNE : public DB_EXCEPTION
/**
* @brief thrown when an output exists, but shouldn't, namely when adding a block
*/
class OUTPUT_EXISTS : public DB_EXCEPTION
{
class OUTPUT_EXISTS : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
OUTPUT_EXISTS() : DB_EXCEPTION("The output to be added already exists!") {}
@ -324,8 +305,7 @@ class OUTPUT_EXISTS : public DB_EXCEPTION
/**
* @brief thrown when a spent key image exists, but shouldn't, namely when adding a block
*/
class KEY_IMAGE_EXISTS : public DB_EXCEPTION
{
class KEY_IMAGE_EXISTS : public DB_EXCEPTION {
public:
using DB_EXCEPTION::DB_EXCEPTION;
KEY_IMAGE_EXISTS() : DB_EXCEPTION("The spent key image to be added already exists!") {}
@ -335,7 +315,6 @@ class KEY_IMAGE_EXISTS : public DB_EXCEPTION
* End of Exception Definitions
***********************************/
/**
* @brief The BlockchainDB backing store interface declaration/contract
*
@ -348,8 +327,7 @@ class KEY_IMAGE_EXISTS : public DB_EXCEPTION
* A subclass which encounters an issue should report that issue by throwing
* a DB_EXCEPTION which adequately conveys the issue.
*/
class BlockchainDB
{
class BlockchainDB {
private:
/*********************************************************************
* private virtual members
@ -372,21 +350,23 @@ private:
* @param coins_generated the number of coins generated total after this block
* @param blk_hash the hash of the block
*/
virtual void add_block( const block& blk
, size_t block_weight
, uint64_t long_term_block_weight
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, uint64_t num_rct_outs
, const crypto::hash& blk_hash
) = 0;
virtual void add_block(
const block& blk,
size_t block_weight,
uint64_t long_term_block_weight,
const difficulty_type& cumulative_difficulty,
const uint64_t& coins_generated,
uint64_t num_rct_outs,
const crypto::hash& blk_hash) = 0;
/**
* @brief remove data about the top block
*
* The subclass implementing this will remove the block data from the top
* block in the chain. The data to be removed is that which was added in
* BlockchainDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash)
* BlockchainDB::add_block(const block& blk, size_t block_weight, uint64_t
* long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t&
* coins_generated, const crypto::hash& blk_hash)
*
* If any of this cannot be done, the subclass should throw the corresponding
* subclass of DB_EXCEPTION
@ -413,7 +393,11 @@ private:
* @param tx_prunable_hash the hash of the prunable part of the transaction
* @return the transaction ID
*/
virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, std::string>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0;
virtual uint64_t add_transaction_data(
const crypto::hash& blk_hash,
const std::pair<transaction, std::string>& tx,
const crypto::hash& tx_hash,
const crypto::hash& tx_prunable_hash) = 0;
/**
* @brief remove data about a transaction
@ -459,7 +443,12 @@ private:
* @param commitment the rct commitment to the output amount
* @return amount output index
*/
virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) = 0;
virtual uint64_t add_output(
const crypto::hash& tx_hash,
const tx_out& tx_output,
const uint64_t& local_index,
const uint64_t unlock_time,
const rct::key* commitment) = 0;
/**
* @brief store amount output indices for a tx's outputs
@ -474,7 +463,8 @@ private:
* @param tx_id ID of the transaction containing these outputs
* @param amount_output_indices the amount output indices of the transaction
*/
virtual void add_tx_amount_output_indices(const uint64_t tx_id, const std::vector<uint64_t>& amount_output_indices) = 0;
virtual void add_tx_amount_output_indices(
const uint64_t tx_id, const std::vector<uint64_t>& amount_output_indices) = 0;
/**
* @brief store a spent key
@ -500,7 +490,6 @@ private:
*/
virtual void remove_spent_key(const crypto::key_image& k_image) = 0;
/*********************************************************************
* private concrete members
*********************************************************************/
@ -527,9 +516,7 @@ private:
std::chrono::nanoseconds time_add_block1 = 0ns; //!< a performance metric
std::chrono::nanoseconds time_add_transaction = 0ns; //!< a performance metric
protected:
/**
* @brief helper function for add_transactions, to add each individual transaction
*
@ -539,16 +526,20 @@ protected:
* @param blk_hash hash of the block which has the transaction
* @param tx the transaction to add
* @param tx_hash_ptr the hash of the transaction, if already calculated
* @param tx_prunable_hash_ptr the hash of the prunable part of the transaction, if already calculated
* @param tx_prunable_hash_ptr the hash of the prunable part of the transaction, if already
* calculated
*/
void add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, std::string>& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL);
void add_transaction(
const crypto::hash& blk_hash,
const std::pair<transaction, std::string>& tx,
const crypto::hash* tx_hash_ptr = NULL,
const crypto::hash* tx_prunable_hash_ptr = NULL);
mutable std::chrono::nanoseconds time_tx_exists = 0ns; //!< a performance metric
std::chrono::nanoseconds time_commit1 = 0ns; //!< a performance metric
bool m_auto_remove_logs = true; //!< whether or not to automatically remove old logs
public:
/**
* @brief An empty constructor.
*/
@ -596,7 +587,8 @@ public:
* @param filename a path referring to the BlockchainDB to open
* @param db_flags flags relevant to how to open/use the BlockchainDB
*/
virtual void open(const fs::path& filename, cryptonote::network_type nettype, const int db_flags = 0) = 0;
virtual void open(
const fs::path& filename, cryptonote::network_type nettype, const int db_flags = 0) = 0;
/**
* @brief Gets the current open/ready state of the BlockchainDB
@ -668,7 +660,8 @@ public:
* which can fail if the existing database file is in an incompatible format.
* As such, this function needs to be called before calling open().
*
* @param folder The path of the folder containing the database file(s) which must not end with slash '/'.
* @param folder The path of the folder containing the database file(s) which must not end
* with slash '/'.
*
* @return true if the operation is succesfull
*/
@ -685,7 +678,6 @@ public:
*/
virtual std::string get_db_name() const = 0;
/**
* @brief acquires the BlockchainDB lock
*
@ -814,23 +806,28 @@ public:
*
* @return the height of the chain post-addition
*/
virtual uint64_t add_block( const std::pair<block, std::string>& blk
, size_t block_weight
, uint64_t long_term_block_weight
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, const std::vector<std::pair<transaction, std::string>>& txs
);
virtual uint64_t add_block(
const std::pair<block, std::string>& blk,
size_t block_weight,
uint64_t long_term_block_weight,
const difficulty_type& cumulative_difficulty,
const uint64_t& coins_generated,
const std::vector<std::pair<transaction, std::string>>& txs);
virtual void update_block_checkpoint(checkpoint_t const& checkpoint) = 0;
virtual void remove_block_checkpoint(uint64_t height) = 0;
virtual bool get_block_checkpoint(uint64_t height, checkpoint_t& checkpoint) const = 0;
virtual bool get_top_checkpoint(checkpoint_t& checkpoint) const = 0;
// num_desired_checkpoints: set to GET_ALL_CHECKPOINTS to collect as many checkpoints as possible
// num_desired_checkpoints: set to GET_ALL_CHECKPOINTS to collect as many checkpoints as
// possible
static constexpr size_t GET_ALL_CHECKPOINTS = 0;
virtual std::vector<checkpoint_t> get_checkpoints_range(uint64_t start, uint64_t end, size_t num_desired_checkpoints = GET_ALL_CHECKPOINTS) const = 0;
virtual bool get_immutable_checkpoint(checkpoint_t *immutable_checkpoint, uint64_t block_height) const;
virtual std::vector<checkpoint_t> get_checkpoints_range(
uint64_t start,
uint64_t end,
size_t num_desired_checkpoints = GET_ALL_CHECKPOINTS) const = 0;
virtual bool get_immutable_checkpoint(
checkpoint_t* immutable_checkpoint, uint64_t block_height) const;
/**
* @brief checks if a block exists
@ -961,7 +958,8 @@ public:
*
* @return the cumulative number of rct outputs
*/
virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const = 0;
virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(
const std::vector<uint64_t>& heights) const = 0;
/**
* @brief fetch the top block's timestamp
@ -1059,7 +1057,8 @@ public:
*
* @return the weights
*/
virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const = 0;
virtual std::vector<uint64_t> get_long_term_block_weights(
uint64_t start_height, size_t count) const = 0;
/**
* @brief fetch a block's hash
@ -1107,14 +1106,16 @@ public:
*
* @return a vector of block hashes
*/
virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const = 0;
virtual std::vector<crypto::hash> get_hashes_range(
const uint64_t& h1, const uint64_t& h2) const = 0;
/**
* @brief fetch the top block's hash
*
* The subclass should return the hash of the most recent block
*
* @param block_height if non NULL, returns the height of that block (ie, the blockchain height minus 1)
* @param block_height if non NULL, returns the height of that block (ie, the blockchain height
* minus 1)
*
* @return the top block's hash
*/
@ -1138,7 +1139,6 @@ public:
*/
virtual uint64_t height() const = 0;
/**
* <!--
* TODO: Rewrite (if necessary) such that all calls to remove_* are
@ -1161,7 +1161,6 @@ public:
*/
virtual void pop_block(block& blk, std::vector<transaction>& txs);
/**
* @brief check if a transaction with a given hash exists
*
@ -1282,7 +1281,8 @@ public:
virtual bool get_pruned_tx_blob(const crypto::hash& h, std::string& tx) const = 0;
/**
* @brief fetches a number of pruned transaction blob from the given hash, in canonical blockchain order
* @brief fetches a number of pruned transaction blob from the given hash, in canonical
* blockchain order
*
* The subclass should return the pruned transactions stored from the one with the given
* hash.
@ -1295,7 +1295,8 @@ public:
*
* @return true iff the transactions were found
*/
virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<std::string> &bd) const = 0;
virtual bool get_pruned_tx_blobs_from(
const crypto::hash& h, size_t count, std::vector<std::string>& bd) const = 0;
/**
* @brief fetches the prunable transaction blob with the given hash
@ -1323,7 +1324,8 @@ public:
*
* @return true iff the transaction was found
*/
virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const = 0;
virtual bool get_prunable_tx_hash(
const crypto::hash& tx_hash, crypto::hash& prunable_hash) const = 0;
/**
* @brief fetches the total number of transactions ever
@ -1379,7 +1381,8 @@ public:
*
* @return vector of heights corresponding to each hash, or (uint64_t)-1 if not found
*/
virtual std::vector<uint64_t> get_tx_block_heights(const std::vector<crypto::hash> &h) const = 0;
virtual std::vector<uint64_t> get_tx_block_heights(
const std::vector<crypto::hash>& h) const = 0;
// returns the total number of outputs of amount <amount>
/**
@ -1420,7 +1423,10 @@ public:
*
* @return the requested output data
*/
virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt = true) const = 0;
virtual output_data_t get_output_key(
const uint64_t& amount,
const uint64_t& index,
bool include_commitmemt = true) const = 0;
/**
* @brief gets an output's tx hash and index
@ -1446,7 +1452,8 @@ public:
*
* @return the tx hash and output index
*/
virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const = 0;
virtual tx_out_index get_output_tx_and_index(
const uint64_t& amount, const uint64_t& index) const = 0;
/**
* @brief gets some outputs' tx hashes and indices
@ -1459,7 +1466,10 @@ public:
* @param offsets a list of amount-specific output indices
* @param indices return-by-reference a list of tx hashes and output indices (as pairs)
*/
virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) const = 0;
virtual void get_output_tx_and_index(
const uint64_t& amount,
const std::vector<uint64_t>& offsets,
std::vector<tx_out_index>& indices) const = 0;
/**
* @brief gets outputs' data
@ -1472,7 +1482,11 @@ public:
* @param offsets a list of amount-specific output indices
* @param outputs return-by-reference a list of outputs' metadata
*/
virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false) const = 0;
virtual void get_output_key(
const epee::span<const uint64_t>& amounts,
const std::vector<uint64_t>& offsets,
std::vector<output_data_t>& outputs,
bool allow_partial = false) const = 0;
/*
* FIXME: Need to check with git blame and ask what this does to
@ -1495,7 +1509,8 @@ public:
*
* @return a list of amount-specific output indices
*/
virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes = 1) const = 0;
virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(
const uint64_t tx_id, size_t n_txes = 1) const = 0;
/**
* @brief check if a key image is stored as spent
@ -1511,7 +1526,8 @@ public:
*
* @param details the details of the transaction to add
*/
virtual void add_txpool_tx(const crypto::hash &txid, const std::string &blob, const txpool_tx_meta_t& details) = 0;
virtual void add_txpool_tx(
const crypto::hash& txid, const std::string& blob, const txpool_tx_meta_t& details) = 0;
/**
* @brief update a txpool transaction's metadata
@ -1620,7 +1636,11 @@ public:
* @param: data: the metadata for the block
* @param: blob: the block's blob
*/
virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const std::string &blob, const std::string *checkpoint) = 0;
virtual void add_alt_block(
const crypto::hash& blkid,
const cryptonote::alt_block_data_t& data,
const std::string& blob,
const std::string* checkpoint) = 0;
/**
* @brief get an alternative block by hash
@ -1631,7 +1651,11 @@ public:
*
* @return true if the block was found in the alternative blocks list, false otherwise
*/
virtual bool get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, std::string *blob, std::string *checkpoint) const = 0;
virtual bool get_alt_block(
const crypto::hash& blkid,
alt_block_data_t* data,
std::string* blob,
std::string* checkpoint) const = 0;
/**
* @brief get the block header from the alternative block db
@ -1643,7 +1667,11 @@ public:
*
* @return true if the block was found in the alternative blocks list, false otherwise
*/
bool get_alt_block_header(const crypto::hash &blkid, alt_block_data_t *data, cryptonote::block_header *header, std::string *checkpoint) const;
bool get_alt_block_header(
const crypto::hash& blkid,
alt_block_data_t* data,
cryptonote::block_header* header,
std::string* checkpoint) const;
/**
* @brief remove an alternative block
@ -1675,7 +1703,10 @@ public:
*
* @return false if the function returns false for any transaction, otherwise true
*/
virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const std::string*)>, bool include_blob = false, bool include_unrelayed_txes = true) const = 0;
virtual bool for_all_txpool_txes(
std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const std::string*)>,
bool include_blob = false,
bool include_unrelayed_txes = true) const = 0;
/**
* @brief runs a function over all key images stored
@ -1710,7 +1741,10 @@ public:
*
* @return false if the function returns false for any block, otherwise true
*/
virtual bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const = 0;
virtual bool for_blocks_range(
const uint64_t& h1,
const uint64_t& h2,
std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const = 0;
/**
* @brief runs a function over all transactions stored
@ -1729,7 +1763,9 @@ public:
*
* @return false if the function returns false for any transaction, otherwise true
*/
virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const = 0;
virtual bool for_all_transactions(
std::function<bool(const crypto::hash&, const cryptonote::transaction&)>,
bool pruned) const = 0;
/**
* @brief runs a function over all outputs stored
@ -1748,8 +1784,12 @@ public:
*
* @return false if the function returns false for any output, otherwise true
*/
virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const = 0;
virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const = 0;
virtual bool for_all_outputs(
std::function<bool(
uint64_t amount, const crypto::hash& tx_hash, uint64_t height, size_t tx_idx)>
f) const = 0;
virtual bool for_all_outputs(
uint64_t amount, const std::function<bool(uint64_t height)>& f) const = 0;
/**
* @brief runs a function over all alternative blocks stored
@ -1767,7 +1807,13 @@ public:
*
* @return false if the function returns false for any output, otherwise true
*/
virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const std::string *block_blob, const std::string *checkpoint_blob)> f, bool include_blob = false) const = 0;
virtual bool for_all_alt_blocks(
std::function<
bool(const crypto::hash& blkid,
const alt_block_data_t& data,
const std::string* block_blob,
const std::string* checkpoint_blob)> f,
bool include_blob = false) const = 0;
/**
* @brief return a histogram of outputs on the blockchain
@ -1779,9 +1825,18 @@ public:
*
* @return a set of amount/instances
*/
virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const = 0;
virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(
const std::vector<uint64_t>& amounts,
bool unlocked,
uint64_t recent_cutoff,
uint64_t min_count) const = 0;
virtual bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const = 0;
virtual bool get_output_distribution(
uint64_t amount,
uint64_t from_height,
uint64_t to_height,
std::vector<uint64_t>& distribution,
uint64_t& base) const = 0;
/**
* @brief is BlockchainDB in read-only mode?
@ -1810,13 +1865,16 @@ public:
/// Updates the given proof data with the latest stored info for the given service node. Returns
/// true if found (and fields updated), false otherwise.
virtual bool get_service_node_proof(const crypto::public_key &pubkey, service_nodes::proof_info &proof) const = 0;
virtual bool get_service_node_proof(
const crypto::public_key& pubkey, service_nodes::proof_info& proof) const = 0;
/// Returns pubkeys and proof data for all currently stored service nodes.
virtual std::unordered_map<crypto::public_key, service_nodes::proof_info> get_all_service_node_proofs() const = 0;
virtual std::unordered_map<crypto::public_key, service_nodes::proof_info>
get_all_service_node_proofs() const = 0;
/// Creates or updates the proof data for a service node from a proof.
virtual void set_service_node_proof(const crypto::public_key &pubkey, const service_nodes::proof_info &proof) = 0;
virtual void set_service_node_proof(
const crypto::public_key& pubkey, const service_nodes::proof_info& proof) = 0;
/// Removes stored serialized proof sn data associated with the given pubkey. Returns true if
/// found, false if not found.
@ -1843,7 +1901,8 @@ public:
// was invoked and loaded historical timestamp/difficulties into (allowing
// this function to be called iteratively on the same input arrays over time).
// This should be set to 0 on the initial call.
void fill_timestamps_and_difficulties_for_pow(cryptonote::network_type nettype,
void fill_timestamps_and_difficulties_for_pow(
cryptonote::network_type nettype,
std::vector<uint64_t>& timestamps,
std::vector<uint64_t>& difficulties,
uint64_t chain_height,
@ -1863,37 +1922,29 @@ public:
}; // class BlockchainDB
class db_txn_guard
{
class db_txn_guard {
public:
db_txn_guard(BlockchainDB& db, bool readonly): db(db), readonly(readonly), active(false)
{
if (readonly)
{
db_txn_guard(BlockchainDB& db, bool readonly) : db(db), readonly(readonly), active(false) {
if (readonly) {
active = db.block_rtxn_start();
}
else
{
} else {
db.block_wtxn_start();
active = true;
}
}
db_txn_guard(BlockchainDB* db, bool readonly) : db_txn_guard(*db, readonly) {}
virtual ~db_txn_guard()
{
virtual ~db_txn_guard() {
if (active)
stop();
}
void stop()
{
void stop() {
if (readonly)
db.block_rtxn_stop();
else
db.block_wtxn_stop();
active = false;
}
void abort()
{
void abort() {
if (readonly)
db.block_rtxn_abort();
else

3314
src/blockchain_db/lmdb/db_lmdb.cpp
View file

File diff suppressed because it is too large Load diff

220
src/blockchain_db/lmdb/db_lmdb.h
View file

@ -26,20 +26,19 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <atomic>
#include <lmdb.h>
#include "blockchain_db/blockchain_db.h"
#include "ringct/rctTypes.h"
#include "common/fs.h"
#include <atomic>
#include <boost/thread/thread.hpp>
#include <boost/thread/tss.hpp>
#include <lmdb.h>
#include "blockchain_db/blockchain_db.h"
#include "common/fs.h"
#include "ringct/rctTypes.h"
#define ENABLE_AUTO_RESIZE
namespace cryptonote
{
namespace cryptonote {
struct mdb_block_info;
@ -48,8 +47,7 @@ struct txindex {
tx_data_t data;
};
struct mdb_txn_cursors
{
struct mdb_txn_cursors {
MDB_cursor* blocks;
MDB_cursor* block_heights;
MDB_cursor* block_info;
@ -81,8 +79,7 @@ struct mdb_txn_cursors
MDB_cursor* properties;
};
struct mdb_rflags
{
struct mdb_rflags {
bool m_rf_txn;
bool m_rf_blocks;
bool m_rf_block_heights;
@ -108,8 +105,7 @@ struct mdb_rflags
bool m_rf_properties;
};
struct mdb_threadinfo
{
struct mdb_threadinfo {
MDB_txn* m_ti_rtxn; // per-thread read txn
mdb_txn_cursors m_ti_rcursors; // per-thread read cursors
mdb_rflags m_ti_rflags; // per-thread read state
@ -117,8 +113,7 @@ struct mdb_threadinfo
~mdb_threadinfo();
};
struct mdb_txn_safe
{
struct mdb_txn_safe {
mdb_txn_safe(const bool check = true);
~mdb_txn_safe();
@ -131,15 +126,9 @@ struct mdb_txn_safe
void abort();
void uncheck();
operator MDB_txn*()
{
return m_txn;
}
operator MDB_txn*() { return m_txn; }
operator MDB_txn**()
{
return &m_txn;
}
operator MDB_txn**() { return &m_txn; }
uint64_t num_active_tx() const;
@ -157,7 +146,6 @@ struct mdb_txn_safe
static std::atomic_flag creation_gate;
};
// If m_batch_active is set, a batch transaction exists beyond this class, such
// as a batch import with verification enabled, or possibly (later) a batch
// network sync.
@ -171,13 +159,13 @@ struct mdb_txn_safe
// A regular network sync without batch writes is expected to open a new read
// transaction, as those lookups are part of the validation done prior to the
// write for block and tx data, so no write transaction is open at the time.
class BlockchainLMDB : public BlockchainDB
{
class BlockchainLMDB : public BlockchainDB {
public:
BlockchainLMDB(bool batch_transactions = true);
~BlockchainLMDB();
void open(const fs::path& filename, cryptonote::network_type nettype, const int mdb_flags=0) override;
void open(const fs::path& filename, cryptonote::network_type nettype, const int mdb_flags = 0)
override;
void close() override;
@ -211,7 +199,8 @@ public:
std::string get_block_blob_from_height(uint64_t height) const override;
std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const override;
std::vector<uint64_t> get_block_cumulative_rct_outputs(
const std::vector<uint64_t>& heights) const override;
uint64_t get_block_timestamp(const uint64_t& height) const override;
@ -229,13 +218,15 @@ public:
uint64_t get_block_long_term_weight(const uint64_t& height) const override;
std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const override;
std::vector<uint64_t> get_long_term_block_weights(
uint64_t start_height, size_t count) const override;
crypto::hash get_block_hash_from_height(const uint64_t& height) const override;
std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const override;
std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const override;
std::vector<crypto::hash> get_hashes_range(
const uint64_t& h1, const uint64_t& h2) const override;
crypto::hash top_block_hash(uint64_t* block_height = NULL) const override;
@ -250,33 +241,50 @@ public:
bool get_tx_blob(const crypto::hash& h, std::string& tx) const override;
bool get_pruned_tx_blob(const crypto::hash& h, std::string& tx) const override;
bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<std::string> &bd) const override;
bool get_pruned_tx_blobs_from(
const crypto::hash& h, size_t count, std::vector<std::string>& bd) const override;
bool get_prunable_tx_blob(const crypto::hash& h, std::string& tx) const override;
bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const override;
bool get_prunable_tx_hash(
const crypto::hash& tx_hash, crypto::hash& prunable_hash) const override;
uint64_t get_tx_count() const override;
std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const override;
std::vector<uint64_t> get_tx_block_heights(const std::vector<crypto::hash>& hlist) const override;
std::vector<uint64_t> get_tx_block_heights(
const std::vector<crypto::hash>& hlist) const override;
uint64_t get_num_outputs(const uint64_t& amount) const override;
output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const override;
void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false) const override;
output_data_t get_output_key(
const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const override;
void get_output_key(
const epee::span<const uint64_t>& amounts,
const std::vector<uint64_t>& offsets,
std::vector<output_data_t>& outputs,
bool allow_partial = false) const override;
tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const override;
void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices,
void get_output_tx_and_index_from_global(
const std::vector<uint64_t>& global_indices,
std::vector<tx_out_index>& tx_out_indices) const;
tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const override;
void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) const override;
tx_out_index get_output_tx_and_index(
const uint64_t& amount, const uint64_t& index) const override;
void get_output_tx_and_index(
const uint64_t& amount,
const std::vector<uint64_t>& offsets,
std::vector<tx_out_index>& indices) const override;
std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes) const override;
std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(
const uint64_t tx_id, size_t n_txes) const override;
bool has_key_image(const crypto::key_image& img) const override;
void add_txpool_tx(const crypto::hash &txid, const std::string &blob, const txpool_tx_meta_t& meta) override;
void add_txpool_tx(
const crypto::hash& txid,
const std::string& blob,
const txpool_tx_meta_t& meta) override;
void update_txpool_tx(const crypto::hash& txid, const txpool_tx_meta_t& meta) override;
uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const override;
bool txpool_has_tx(const crypto::hash& txid) const override;
@ -289,33 +297,63 @@ public:
bool update_pruning() override;
bool check_pruning() override;
void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const std::string &blob, const std::string *checkpoint) override;
bool get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, std::string *blob, std::string *checkpoint) const override;
void add_alt_block(
const crypto::hash& blkid,
const cryptonote::alt_block_data_t& data,
const std::string& blob,
const std::string* checkpoint) override;
bool get_alt_block(
const crypto::hash& blkid,
alt_block_data_t* data,
std::string* blob,
std::string* checkpoint) const override;
void remove_alt_block(const crypto::hash& blkid) override;
uint64_t get_alt_block_count() override;
void drop_alt_blocks() override;
bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const std::string*)> f, bool include_blob = false, bool include_unrelayed_txes = true) const override;
bool for_all_txpool_txes(
std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const std::string*)> f,
bool include_blob = false,
bool include_unrelayed_txes = true) const override;
bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const override;
bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const override;
bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const override;
bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const override;
bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const override;
bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const std::string *block_blob, const std::string *checkpoint_blob)> f, bool include_blob = false) const override;
bool for_blocks_range(
const uint64_t& h1,
const uint64_t& h2,
std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>)
const override;
bool for_all_transactions(
std::function<bool(const crypto::hash&, const cryptonote::transaction&)>,
bool pruned) const override;
bool for_all_outputs(
std::function<bool(
uint64_t amount, const crypto::hash& tx_hash, uint64_t height, size_t tx_idx)>
f) const override;
bool for_all_outputs(
uint64_t amount, const std::function<bool(uint64_t height)>& f) const override;
bool for_all_alt_blocks(
std::function<
bool(const crypto::hash& blkid,
const alt_block_data_t& data,
const std::string* block_blob,
const std::string* checkpoint_blob)> f,
bool include_blob = false) const override;
uint64_t add_block( const std::pair<block, std::string>& blk
, size_t block_weight
, uint64_t long_term_block_weight
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, const std::vector<std::pair<transaction, std::string>>& txs
) override;
uint64_t add_block(
const std::pair<block, std::string>& blk,
size_t block_weight,
uint64_t long_term_block_weight,
const difficulty_type& cumulative_difficulty,
const uint64_t& coins_generated,
const std::vector<std::pair<transaction, std::string>>& txs) override;
void update_block_checkpoint(checkpoint_t const& checkpoint) override;
void remove_block_checkpoint(uint64_t height) override;
bool get_block_checkpoint(uint64_t height, checkpoint_t& checkpoint) const override;
bool get_top_checkpoint(checkpoint_t& checkpoint) const override;
std::vector<checkpoint_t> get_checkpoints_range(uint64_t start, uint64_t end, size_t num_desired_checkpoints = GET_ALL_CHECKPOINTS) const override;
std::vector<checkpoint_t> get_checkpoints_range(
uint64_t start,
uint64_t end,
size_t num_desired_checkpoints = GET_ALL_CHECKPOINTS) const override;
void set_batch_transactions(bool batch_transactions) override;
bool batch_start(uint64_t batch_num_blocks = 0, uint64_t batch_bytes = 0) override;
@ -346,9 +384,18 @@ public:
*
* @return a set of amount/instances
*/
std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const override;
std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(
const std::vector<uint64_t>& amounts,
bool unlocked,
uint64_t recent_cutoff,
uint64_t min_count) const override;
bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const override;
bool get_output_distribution(
uint64_t amount,
uint64_t from_height,
uint64_t to_height,
std::vector<uint64_t>& distribution,
uint64_t& base) const override;
void get_output_blacklist(std::vector<uint64_t>& blacklist) const override;
void add_output_blacklist(std::vector<uint64_t> const& blacklist) override;
@ -364,31 +411,34 @@ private:
void check_and_resize_for_batch(uint64_t batch_num_blocks, uint64_t batch_bytes);
uint64_t get_estimated_batch_size(uint64_t batch_num_blocks, uint64_t batch_bytes) const;
void add_block( const block& blk
, size_t block_weight
, uint64_t long_term_block_weight
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, uint64_t num_rct_outs
, const crypto::hash& block_hash
) override;
void add_block(
const block& blk,
size_t block_weight,
uint64_t long_term_block_weight,
const difficulty_type& cumulative_difficulty,
const uint64_t& coins_generated,
uint64_t num_rct_outs,
const crypto::hash& block_hash) override;
void remove_block() override;
uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, std::string>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) override;
uint64_t add_transaction_data(
const crypto::hash& blk_hash,
const std::pair<transaction, std::string>& tx,
const crypto::hash& tx_hash,
const crypto::hash& tx_prunable_hash) override;
void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) override;
uint64_t add_output(const crypto::hash& tx_hash,
uint64_t add_output(
const crypto::hash& tx_hash,
const tx_out& tx_output,
const uint64_t& local_index,
const uint64_t unlock_time,
const rct::key *commitment
) override;
const rct::key* commitment) override;
void add_tx_amount_output_indices(const uint64_t tx_id,
const std::vector<uint64_t>& amount_output_indices
) override;
void add_tx_amount_output_indices(
const uint64_t tx_id, const std::vector<uint64_t>& amount_output_indices) override;
void remove_tx_outputs(const uint64_t tx_id, const transaction& tx);
@ -410,7 +460,8 @@ private:
uint64_t get_database_size() const override;
std::vector<uint64_t> get_block_info_64bit_fields(uint64_t start_height, size_t count, uint64_t (*extract)(const mdb_block_info*)) const;
std::vector<uint64_t> get_block_info_64bit_fields(
uint64_t start_height, size_t count, uint64_t (*extract)(const mdb_block_info*)) const;
uint64_t get_max_block_size() override;
void add_max_block_size(uint64_t sz) override;
@ -431,21 +482,28 @@ private:
void cleanup_batch();
bool get_block_checkpoint_internal(uint64_t height, checkpoint_t &checkpoint, MDB_cursor_op op) const;
bool get_block_checkpoint_internal(
uint64_t height, checkpoint_t& checkpoint, MDB_cursor_op op) const;
void set_service_node_data(const std::string& data, bool long_term) override;
bool get_service_node_data(std::string& data, bool long_term) const override;
void clear_service_node_data() override;
bool get_service_node_proof(const crypto::public_key& pubkey, service_nodes::proof_info& proof) const override;
void set_service_node_proof(const crypto::public_key& pubkey, const service_nodes::proof_info& proof) override;
std::unordered_map<crypto::public_key, service_nodes::proof_info> get_all_service_node_proofs() const override;
bool get_service_node_proof(
const crypto::public_key& pubkey, service_nodes::proof_info& proof) const override;
void set_service_node_proof(
const crypto::public_key& pubkey, const service_nodes::proof_info& proof) override;
std::unordered_map<crypto::public_key, service_nodes::proof_info> get_all_service_node_proofs()
const override;
bool remove_service_node_proof(const crypto::public_key& pubkey) override;
private:
template <typename T,
std::enable_if_t<std::is_same_v<T, cryptonote::block> ||
template <
typename T,
std::enable_if_t<
std::is_same_v<T, cryptonote::block> ||
std::is_same_v<T, cryptonote::block_header> ||
std::is_same_v<T, std::string>, int> = 0>
std::is_same_v<T, std::string>,
int> = 0>
T get_and_convert_block_blob_from_height(uint64_t height) const;
MDB_env* m_env;

32
src/blockchain_db/locked_txn.h
View file

@ -32,27 +32,43 @@
#include "cryptonote_core/blockchain.h"
namespace cryptonote
{
namespace cryptonote {
// This class is meant to create a batch when none currently exists.
// If a batch exists, it can't be from another thread, since we can
// only be called with the txpool lock taken, and it is held during
// the whole prepare/handle/cleanup incoming block sequence.
class LockedTXN {
public:
LockedTXN(Blockchain &b): m_db{b.get_db()} {
m_batch = m_db.batch_start();
}
LockedTXN(Blockchain& b) : m_db{b.get_db()} { m_batch = m_db.batch_start(); }
LockedTXN(const LockedTXN&) = delete;
LockedTXN& operator=(const LockedTXN&) = delete;
LockedTXN(LockedTXN&& o) : m_db{o.m_db}, m_batch{o.m_batch} { o.m_batch = false; }
LockedTXN& operator=(LockedTXN&&) = delete;
void commit() { try { if (m_batch) { m_db.batch_stop(); m_batch = false; } } catch (const std::exception &e) { log::warning(globallogcat, "LockedTXN::commit filtering exception: {}", e.what()); } }
void abort() { try { if (m_batch) { m_db.batch_abort(); m_batch = false; } } catch (const std::exception &e) { log::warning(globallogcat, "LockedTXN::abort filtering exception: {}", e.what()); } }
void commit() {
try {
if (m_batch) {
m_db.batch_stop();
m_batch = false;
}
} catch (const std::exception& e) {
log::warning(globallogcat, "LockedTXN::commit filtering exception: {}", e.what());
}
}
void abort() {
try {
if (m_batch) {
m_db.batch_abort();
m_batch = false;
}
} catch (const std::exception& e) {
log::warning(globallogcat, "LockedTXN::abort filtering exception: {}", e.what());
}
}
~LockedTXN() { this->abort(); }
private:
BlockchainDB& m_db;
bool m_batch;
};
}
} // namespace cryptonote

294
src/blockchain_db/sqlite/db_sqlite.cpp
View file

@ -27,27 +27,29 @@
#include "db_sqlite.h"
#include <sqlite3.h>
#include <sodium.h>
#include <fmt/core.h>
#include <sodium.h>
#include <sqlite3.h>
#include <cassert>
#include "common/string_util.h"
#include "cryptonote_basic/hardfork.h"
#include "cryptonote_config.h"
#include "cryptonote_core/blockchain.h"
#include "cryptonote_core/service_node_list.h"
#include "common/string_util.h"
#include "cryptonote_basic/hardfork.h"
namespace cryptonote {
static auto logcat = log::Cat("blockchain.db.sqlite");
BlockchainSQLite::BlockchainSQLite(cryptonote::network_type nettype, fs::path db_path): db::Database(db_path, ""), m_nettype(nettype), filename {db_path.u8string()} {
BlockchainSQLite::BlockchainSQLite(cryptonote::network_type nettype, fs::path db_path) :
db::Database(db_path, ""), m_nettype(nettype), filename{db_path.u8string()} {
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
height = 0;
if (!db.tableExists("batched_payments_accrued") || !db.tableExists("batched_payments_raw") || !db.tableExists("batch_db_info")) {
if (!db.tableExists("batched_payments_accrued") || !db.tableExists("batched_payments_raw") ||
!db.tableExists("batch_db_info")) {
create_schema();
}
@ -61,7 +63,8 @@ namespace cryptonote {
auto& netconf = cryptonote::get_config(m_nettype);
db.exec(fmt::format(R"(
db.exec(fmt::format(
R"(
CREATE TABLE batched_payments_accrued(
address VARCHAR NOT NULL,
amount BIGINT NOT NULL,
@ -131,12 +134,10 @@ namespace cryptonote {
if (!have_offset) {
log::info(logcat, "Adding payout_offset to batching db");
auto& netconf = get_config(m_nettype);
SQLite::Transaction transaction{
db,
SQLite::TransactionBehavior::IMMEDIATE
};
SQLite::Transaction transaction{db, SQLite::TransactionBehavior::IMMEDIATE};
db.exec(fmt::format(R"(
db.exec(fmt::format(
R"(
ALTER TABLE batched_payments_accrued ADD COLUMN payout_offset INTEGER NOT NULL DEFAULT -1;
CREATE INDEX batched_payments_accrued_payout_offset_idx ON batched_payments_accrued(payout_offset);
@ -148,10 +149,13 @@ namespace cryptonote {
INSERT INTO batched_payments_accrued(address, payout_offset, amount) VALUES(OLD.address, OLD.height_paid % {}, OLD.amount)
ON CONFLICT(address) DO UPDATE SET amount = (amount + excluded.amount);
END;
)", netconf.BATCHING_INTERVAL));
)",
netconf.BATCHING_INTERVAL));
auto st = prepared_st("UPDATE batched_payments_accrued SET payout_offset = ? WHERE address = ?");
for (const auto& address : prepared_results<std::string>("SELECT address from batched_payments_accrued")) {
auto st = prepared_st(
"UPDATE batched_payments_accrued SET payout_offset = ? WHERE address = ?");
for (const auto& address : prepared_results<std::string>("SELECT address from "
"batched_payments_accrued")) {
cryptonote::address_parse_info addr_info{};
cryptonote::get_account_address_from_str(addr_info, m_nettype, address);
auto offset = static_cast<int>(addr_info.address.modulus(netconf.BATCHING_INTERVAL));
@ -159,11 +163,14 @@ namespace cryptonote {
st->reset();
}
auto count = prepared_get<int>("SELECT COUNT(*) FROM batched_payments_accrued WHERE payout_offset NOT BETWEEN 0 AND ?",
auto count = prepared_get<int>(
"SELECT COUNT(*) FROM batched_payments_accrued WHERE payout_offset NOT BETWEEN 0 "
"AND ?",
static_cast<int>(netconf.BATCHING_INTERVAL));
if (count != 0) {
constexpr auto error = "Batching db update to add offsets failed: not all addresses were converted";
constexpr auto error =
"Batching db update to add offsets failed: not all addresses were converted";
log::error(logcat, error);
throw std::runtime_error{error};
}
@ -171,16 +178,15 @@ namespace cryptonote {
transaction.commit();
}
const auto archive_table_count = prepared_get<int64_t>("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='batched_payments_accrued_archive';");
if(archive_table_count == 0)
{
const auto archive_table_count = prepared_get<int64_t>(
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND "
"name='batched_payments_accrued_archive';");
if (archive_table_count == 0) {
log::info(logcat, "Adding archiving to batching db");
auto& netconf = get_config(m_nettype);
SQLite::Transaction transaction{
db,
SQLite::TransactionBehavior::IMMEDIATE
};
db.exec(fmt::format(R"(
SQLite::Transaction transaction{db, SQLite::TransactionBehavior::IMMEDIATE};
db.exec(fmt::format(
R"(
CREATE TABLE batched_payments_accrued_archive(
address VARCHAR NOT NULL,
amount BIGINT NOT NULL,
@ -204,7 +210,9 @@ namespace cryptonote {
FOR EACH ROW WHEN NEW.height < OLD.height BEGIN
DELETE FROM batched_payments_accrued_archive WHERE archive_height >= NEW.height;
END;
)", netconf.STORE_LONG_TERM_STATE_INTERVAL, 500));
)",
netconf.STORE_LONG_TERM_STATE_INTERVAL,
500));
transaction.commit();
}
}
@ -232,9 +240,7 @@ namespace cryptonote {
void BlockchainSQLite::update_height(uint64_t new_height) {
log::trace(logcat, "BlockchainDB_SQLITE::{} Called with new height: {}", __func__, new_height);
height = new_height;
prepared_exec(
"UPDATE batch_db_info SET height = ?",
static_cast<int64_t>(height));
prepared_exec("UPDATE batch_db_info SET height = ?", static_cast<int64_t>(height));
}
void BlockchainSQLite::increment_height() {
@ -247,17 +253,16 @@ namespace cryptonote {
update_height(height - 1);
}
void BlockchainSQLite::blockchain_detached(uint64_t new_height)
{
void BlockchainSQLite::blockchain_detached(uint64_t new_height) {
if (height < new_height)
return;
int64_t revert_to_height = new_height - 1;
auto maybe_prev_interval = prepared_maybe_get<int64_t>(
"SELECT DISTINCT archive_height FROM batched_payments_accrued_archive WHERE archive_height <= ? ORDER BY archive_height DESC LIMIT 1",
"SELECT DISTINCT archive_height FROM batched_payments_accrued_archive WHERE "
"archive_height <= ? ORDER BY archive_height DESC LIMIT 1",
revert_to_height);
if (!maybe_prev_interval)
{
if (!maybe_prev_interval) {
auto fork_height = cryptonote::get_hard_fork_heights(m_nettype, hf::hf19_reward_batching);
reset_database();
update_height(fork_height.first.value_or(0));
@ -265,7 +270,8 @@ namespace cryptonote {
}
const auto prev_interval = *maybe_prev_interval;
db.exec(fmt::format(R"(
db.exec(fmt::format(
R"(
DELETE FROM batched_payments_raw WHERE height_paid > {0};
DELETE FROM batched_payments_accrued;
@ -275,21 +281,20 @@ namespace cryptonote {
FROM batched_payments_accrued_archive WHERE archive_height = {0};
DELETE FROM batched_payments_accrued_archive WHERE archive_height >= {0};
)", prev_interval));
)",
prev_interval));
update_height(prev_interval);
return;
}
// Must be called with the address_str_cache_mutex held!
const std::string& BlockchainSQLite::get_address_str(const account_public_address& addr)
{
const std::string& BlockchainSQLite::get_address_str(const account_public_address& addr) {
auto& address_str = address_str_cache[addr];
if (address_str.empty())
address_str = cryptonote::get_account_address_as_str(m_nettype, 0, addr);
return address_str;
}
bool BlockchainSQLite::add_sn_rewards(const std::vector<cryptonote::batch_sn_payment>& payments) {
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
auto insert_payment = prepared_st(
@ -299,10 +304,15 @@ namespace cryptonote {
const auto& netconf = get_config(m_nettype);
for (auto& payment : payments) {
auto offset = static_cast<int>(payment.address_info.address.modulus(netconf.BATCHING_INTERVAL));
auto offset =
static_cast<int>(payment.address_info.address.modulus(netconf.BATCHING_INTERVAL));
auto amt = static_cast<int64_t>(payment.amount);
const auto& address_str = get_address_str(payment.address_info.address);
log::trace(logcat, "Adding record for SN reward contributor {} to database with amount {}", address_str, amt);
log::trace(
logcat,
"Adding record for SN reward contributor {} to database with amount {}",
address_str,
amt);
db::exec_query(insert_payment, address_str, offset, amt);
insert_payment->reset();
}
@ -310,16 +320,21 @@ namespace cryptonote {
return true;
}
bool BlockchainSQLite::subtract_sn_rewards(const std::vector<cryptonote::batch_sn_payment>& payments) {
bool BlockchainSQLite::subtract_sn_rewards(
const std::vector<cryptonote::batch_sn_payment>& payments) {
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
auto update_payment = prepared_st(
"UPDATE batched_payments_accrued SET amount = (amount - ?) WHERE address = ?");
for (auto& payment : payments) {
const auto& address_str = get_address_str(payment.address_info.address);
auto result = db::exec_query(update_payment, static_cast<int64_t>(payment.amount), address_str);
auto result =
db::exec_query(update_payment, static_cast<int64_t>(payment.amount), address_str);
if (!result) {
log::error(logcat, "tried to subtract payment from an address that doesn't exist: {}", address_str);
log::error(
logcat,
"tried to subtract payment from an address that doesn't exist: {}",
address_str);
return false;
}
update_payment->reset();
@ -334,13 +349,16 @@ namespace cryptonote {
// <= here because we might have crap in the db that we don't clear until we actually add the HF
// block later on. (This is a pretty slim edge case that happened on devnet and is probably
// virtually impossible on mainnet).
if (m_nettype != cryptonote::network_type::FAKECHAIN && block_height <= cryptonote::get_hard_fork_heights(m_nettype, hf::hf19_reward_batching).first.value_or(0))
if (m_nettype != cryptonote::network_type::FAKECHAIN &&
block_height <= cryptonote::get_hard_fork_heights(m_nettype, hf::hf19_reward_batching)
.first.value_or(0))
return {};
const auto& conf = get_config(m_nettype);
auto accrued_amounts = prepared_results<std::string_view, int64_t>(
"SELECT address, amount FROM batched_payments_accrued WHERE payout_offset = ? AND amount >= ? ORDER BY address ASC",
"SELECT address, amount FROM batched_payments_accrued WHERE payout_offset = ? AND "
"amount >= ? ORDER BY address ASC",
static_cast<int>(block_height % conf.BATCHING_INTERVAL),
static_cast<int64_t>(conf.MIN_BATCH_PAYMENT_AMOUNT * BATCH_REWARD_FACTOR));
@ -349,31 +367,31 @@ namespace cryptonote {
for (auto [address, amount] : accrued_amounts) {
auto& p = payments.emplace_back();
p.amount = amount / BATCH_REWARD_FACTOR * BATCH_REWARD_FACTOR; /* truncate to atomic OXEN */
[[maybe_unused]] bool addr_ok = cryptonote::get_account_address_from_str(p.address_info, m_nettype, address);
[[maybe_unused]] bool addr_ok =
cryptonote::get_account_address_from_str(p.address_info, m_nettype, address);
assert(addr_ok);
}
return payments;
}
uint64_t BlockchainSQLite::get_accrued_earnings(const std::string& address) {
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
auto earnings = prepared_maybe_get<int64_t>(
"SELECT amount FROM batched_payments_accrued WHERE address = ?",
address);
"SELECT amount FROM batched_payments_accrued WHERE address = ?", address);
return static_cast<uint64_t>(earnings.value_or(0) / 1000);
}
std::pair<std::vector<std::string>, std::vector<uint64_t>> BlockchainSQLite::get_all_accrued_earnings() {
std::pair<std::vector<std::string>, std::vector<uint64_t>>
BlockchainSQLite::get_all_accrued_earnings() {
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
std::pair<std::vector<std::string>, std::vector<uint64_t>> result;
auto& [addresses, amounts] = result;
for (auto [addr, amt] : prepared_results<std::string, int64_t>(
"SELECT address, amount FROM batched_payments_accrued")) {
for (auto [addr, amt] : prepared_results<std::string, int64_t>("SELECT address, amount FROM "
"batched_payments_accrued")) {
auto amount = static_cast<uint64_t>(amt / 1000);
if (amount > 0) {
addresses.push_back(std::move(addr));
@ -384,12 +402,17 @@ namespace cryptonote {
return result;
}
void BlockchainSQLite::calculate_rewards(hf hf_version, uint64_t distribution_amount, const service_nodes::service_node_info& sn_info, std::vector<cryptonote::batch_sn_payment>& payments) {
void BlockchainSQLite::calculate_rewards(
hf hf_version,
uint64_t distribution_amount,
const service_nodes::service_node_info& sn_info,
std::vector<cryptonote::batch_sn_payment>& payments) {
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
// Find out how much is due for the operator: fee_portions/PORTIONS * reward
assert(sn_info.portions_for_operator <= old::STAKING_PORTIONS);
uint64_t operator_fee = mul128_div64(sn_info.portions_for_operator, distribution_amount, old::STAKING_PORTIONS);
uint64_t operator_fee =
mul128_div64(sn_info.portions_for_operator, distribution_amount, old::STAKING_PORTIONS);
assert(operator_fee <= distribution_amount);
@ -406,8 +429,10 @@ namespace cryptonote {
[](auto&& a, auto&& b) { return a + b.amount; });
for (auto& contributor : sn_info.contributors) {
// This calculates (contributor.amount / total_contributed_to_winner_sn) * (distribution_amount - operator_fee) but using 128 bit integer math
uint64_t c_reward = mul128_div64(contributor.amount, distribution_amount - operator_fee, total_contributed_to_sn);
// This calculates (contributor.amount / total_contributed_to_winner_sn) *
// (distribution_amount - operator_fee) but using 128 bit integer math
uint64_t c_reward = mul128_div64(
contributor.amount, distribution_amount - operator_fee, total_contributed_to_sn);
if (c_reward > 0)
payments.emplace_back(contributor.address, c_reward);
}
@ -418,12 +443,11 @@ namespace cryptonote {
bool BlockchainSQLite::reward_handler(
const cryptonote::block& block,
const service_nodes::service_node_list::state_t& service_nodes_state,
bool add)
{
bool add) {
// The method we call do actually handle the change: either `add_sn_payments` if add is true,
// `subtract_sn_payments` otherwise:
bool (BlockchainSQLite::* add_or_subtract)(const std::vector<cryptonote::batch_sn_payment>&)
= add ? &BlockchainSQLite::add_sn_rewards : &BlockchainSQLite::subtract_sn_rewards;
bool (BlockchainSQLite::*add_or_subtract)(const std::vector<cryptonote::batch_sn_payment>&) =
add ? &BlockchainSQLite::add_sn_rewards : &BlockchainSQLite::subtract_sn_rewards;
// From here on we calculate everything in milli-atomic OXEN (i.e. thousanths of an atomic
// OXEN) so that our integer math has minimal loss from integer division.
@ -431,7 +455,8 @@ namespace cryptonote {
throw std::logic_error{"Reward distribution amount is too large"};
uint64_t block_reward = block.reward * BATCH_REWARD_FACTOR;
uint64_t service_node_reward = cryptonote::service_node_reward_formula(0, block.major_version) * BATCH_REWARD_FACTOR;
uint64_t service_node_reward =
cryptonote::service_node_reward_formula(0, block.major_version) * BATCH_REWARD_FACTOR;
std::vector<cryptonote::batch_sn_payment> payments;
@ -443,15 +468,16 @@ namespace cryptonote {
std::lock_guard a_s_lock{address_str_cache_mutex};
if (uint64_t tx_fees = block_reward - service_node_reward;
tx_fees > 0
&& block.service_node_winner_key // "service_node_winner_key" tracks the pulse winner; 0 if a mined block
&& crypto_core_ed25519_is_valid_point(block.service_node_winner_key.data())
) {
tx_fees > 0 && block.service_node_winner_key // "service_node_winner_key" tracks the pulse
// winner; 0 if a mined block
&& crypto_core_ed25519_is_valid_point(block.service_node_winner_key.data())) {
if (auto service_node_winner = service_nodes_state.service_nodes_infos.find(block.service_node_winner_key);
if (auto service_node_winner =
service_nodes_state.service_nodes_infos.find(block.service_node_winner_key);
service_node_winner != service_nodes_state.service_nodes_infos.end()) {
calculate_rewards(block.major_version, tx_fees, *service_node_winner->second, payments);
// Takes the block producer and adds its contributors to the batching database for the transaction fees
// Takes the block producer and adds its contributors to the batching database for the
// transaction fees
if (!(this->*add_or_subtract)(payments))
return false;
}
@ -459,14 +485,20 @@ namespace cryptonote {
auto block_height = get_block_height(block);
// Step 2: Iterate over the whole service node list and pay each node 1/service_node_list fraction
const auto payable_service_nodes = service_nodes_state.payable_service_nodes_infos(block_height, m_nettype);
// Step 2: Iterate over the whole service node list and pay each node 1/service_node_list
// fraction
const auto payable_service_nodes =
service_nodes_state.payable_service_nodes_infos(block_height, m_nettype);
size_t total_service_nodes_payable = payable_service_nodes.size();
for (const auto& [node_pubkey, node_info] : payable_service_nodes) {
auto payable_service_node = service_nodes_state.service_nodes_infos.find(node_pubkey);
if (payable_service_node == service_nodes_state.service_nodes_infos.end())
continue;
calculate_rewards(block.major_version, service_node_reward / total_service_nodes_payable, * payable_service_node -> second, payments);
calculate_rewards(
block.major_version,
service_node_reward / total_service_nodes_payable,
*payable_service_node->second,
payments);
// Takes the node and adds its contributors to the batching database
if (!(this->*add_or_subtract)(payments))
return false;
@ -475,11 +507,15 @@ namespace cryptonote {
// Step 3: Add Governance reward to the list
if (m_nettype != cryptonote::network_type::FAKECHAIN) {
if (parsed_governance_addr.first != block.major_version) {
cryptonote::get_account_address_from_str(parsed_governance_addr.second, m_nettype,
cryptonote::get_config(m_nettype).governance_wallet_address(block.major_version));
cryptonote::get_account_address_from_str(
parsed_governance_addr.second,
m_nettype,
cryptonote::get_config(m_nettype).governance_wallet_address(
block.major_version));
parsed_governance_addr.first = block.major_version;
}
uint64_t foundation_reward = cryptonote::governance_reward_formula(block.major_version) * BATCH_REWARD_FACTOR;
uint64_t foundation_reward =
cryptonote::governance_reward_formula(block.major_version) * BATCH_REWARD_FACTOR;
payments.clear();
payments.emplace_back(parsed_governance_addr.second.address, foundation_reward);
if (!(this->*add_or_subtract)(payments))
@ -489,7 +525,8 @@ namespace cryptonote {
return true;
}
bool BlockchainSQLite::add_block(const cryptonote::block& block,
bool BlockchainSQLite::add_block(
const cryptonote::block& block,
const service_nodes::service_node_list::state_t& service_nodes_state) {
auto block_height = get_block_height(block);
log::trace(logcat, "BlockchainDB_SQLITE::{} called on height: {}", __func__, block_height);
@ -508,28 +545,31 @@ namespace cryptonote {
}
if (block_height != height + 1) {
log::error(logcat, "Block height ({}) out of sync with batching database ({})", block_height, height);
log::error(
logcat,
"Block height ({}) out of sync with batching database ({})",
block_height,
height);
return false;
}
// We query our own database as a source of truth to verify the blocks payments against. The calculated_rewards
// variable contains a known good list of who should have been paid in this block
// We query our own database as a source of truth to verify the blocks payments against. The
// calculated_rewards variable contains a known good list of who should have been paid in this
// block
auto calculated_rewards = get_sn_payments(block_height);
// We iterate through the block's coinbase payments and build a copy of our own list of the payments
// miner_tx_vouts this will be compared against calculated_rewards and if they match we know the block is
// paying the correct people only.
// We iterate through the block's coinbase payments and build a copy of our own list of the
// payments miner_tx_vouts this will be compared against calculated_rewards and if they match we
// know the block is paying the correct people only.
std::vector<std::pair<crypto::public_key, uint64_t>> miner_tx_vouts;
for (auto& vout : block.miner_tx.vout)
miner_tx_vouts.emplace_back(var::get<txout_to_key>(vout.target).key, vout.amount);
try {
SQLite::Transaction transaction {
db,
SQLite::TransactionBehavior::IMMEDIATE
};
SQLite::Transaction transaction{db, SQLite::TransactionBehavior::IMMEDIATE};
// Goes through the miner transactions vouts checks they are right and marks them as paid in the database
// Goes through the miner transactions vouts checks they are right and marks them as paid in
// the database
if (!validate_batch_payment(miner_tx_vouts, calculated_rewards, block_height)) {
return false;
}
@ -547,7 +587,8 @@ namespace cryptonote {
return true;
}
bool BlockchainSQLite::pop_block(const cryptonote::block& block,
bool BlockchainSQLite::pop_block(
const cryptonote::block& block,
const service_nodes::service_node_list::state_t& service_nodes_state) {
auto block_height = get_block_height(block);
@ -569,11 +610,7 @@ namespace cryptonote {
}
try {
SQLite::Transaction transaction {
db,
SQLite::TransactionBehavior::IMMEDIATE
};
SQLite::Transaction transaction{db, SQLite::TransactionBehavior::IMMEDIATE};
if (!reward_handler(block, service_nodes_state, /*add=*/false))
return false;
@ -597,7 +634,11 @@ namespace cryptonote {
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
if (miner_tx_vouts.size() != calculated_payments_from_batching_db.size()) {
log::error(logcat, "Length of batch payments ({}) does not match block vouts ({})", calculated_payments_from_batching_db.size(), miner_tx_vouts.size());
log::error(
logcat,
"Length of batch payments ({}) does not match block vouts ({})",
calculated_payments_from_batching_db.size(),
miner_tx_vouts.size());
return false;
}
@ -609,18 +650,26 @@ namespace cryptonote {
[](auto&& a, auto&& b) { return a + b.amount; });
uint64_t total_oxen_payout_in_vouts = 0;
std::vector<batch_sn_payment> finalised_payments;
cryptonote::keypair
const deterministic_keypair = cryptonote::get_deterministic_keypair_from_height(block_height);
cryptonote::keypair const deterministic_keypair =
cryptonote::get_deterministic_keypair_from_height(block_height);
for (size_t vout_index = 0; vout_index < miner_tx_vouts.size(); vout_index++) {
const auto& [pubkey, amt] = miner_tx_vouts[vout_index];
uint64_t amount = amt * BATCH_REWARD_FACTOR;
const auto& from_db = calculated_payments_from_batching_db[vout_index];
if (amount != from_db.amount) {
log::error(logcat, "Batched payout amount incorrect. Should be {}, not {}", from_db.amount, amount);
log::error(
logcat,
"Batched payout amount incorrect. Should be {}, not {}",
from_db.amount,
amount);
return false;
}
crypto::public_key out_eph_public_key{};
if (!cryptonote::get_deterministic_output_key(from_db.address_info.address, deterministic_keypair, vout_index, out_eph_public_key)) {
if (!cryptonote::get_deterministic_output_key(
from_db.address_info.address,
deterministic_keypair,
vout_index,
out_eph_public_key)) {
log::error(logcat, "Failed to generate output one-time public key");
return false;
}
@ -632,18 +681,22 @@ namespace cryptonote {
finalised_payments.emplace_back(from_db.address_info, amount);
}
if (total_oxen_payout_in_vouts != total_oxen_payout_in_our_db) {
log::error(logcat, "Total batched payout amount incorrect. Should be {}, not {}", total_oxen_payout_in_our_db, total_oxen_payout_in_vouts);
log::error(
logcat,
"Total batched payout amount incorrect. Should be {}, not {}",
total_oxen_payout_in_our_db,
total_oxen_payout_in_vouts);
return false;
}
return save_payments(block_height, finalised_payments);
}
bool BlockchainSQLite::save_payments(uint64_t block_height, const std::vector<batch_sn_payment>& paid_amounts) {
bool BlockchainSQLite::save_payments(
uint64_t block_height, const std::vector<batch_sn_payment>& paid_amounts) {
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
auto select_sum = prepared_st(
"SELECT amount from batched_payments_accrued WHERE address = ?");
auto select_sum = prepared_st("SELECT amount from batched_payments_accrued WHERE address = ?");
auto update_paid = prepared_st(
"INSERT INTO batched_payments_paid (address, amount, height_paid) VALUES (?,?,?)");
@ -652,22 +705,37 @@ namespace cryptonote {
for (const auto& payment : paid_amounts) {
const auto& address_str = get_address_str(payment.address_info.address);
if (auto maybe_amount = db::exec_and_maybe_get<int64_t>(select_sum, address_str))
{
if (auto maybe_amount = db::exec_and_maybe_get<int64_t>(select_sum, address_str)) {
// Truncate the thousanths amount to an atomic OXEN:
auto amount = static_cast<uint64_t>(*maybe_amount) / BATCH_REWARD_FACTOR * BATCH_REWARD_FACTOR;
auto amount = static_cast<uint64_t>(*maybe_amount) / BATCH_REWARD_FACTOR *
BATCH_REWARD_FACTOR;
if (amount != payment.amount) {
log::error(logcat, "Invalid amounts passed in to save payments for address {}: received {}, expected {} (truncated from {})", address_str, payment.amount, amount, *maybe_amount);
log::error(
logcat,
"Invalid amounts passed in to save payments for address {}: received {}, "
"expected {} (truncated from {})",
address_str,
payment.amount,
amount,
*maybe_amount);
return false;
}
db::exec_query(update_paid, address_str, static_cast<int64_t>(amount), static_cast<int64_t>(block_height));
db::exec_query(
update_paid,
address_str,
static_cast<int64_t>(amount),
static_cast<int64_t>(block_height));
update_paid->reset();
}
else {
// This shouldn't occur: we validate payout addresses much earlier in the block validation.
log::error(logcat, "Internal error: Invalid amounts passed in to save payments for address {}: that address has no accrued rewards", address_str);
} else {
// This shouldn't occur: we validate payout addresses much earlier in the block
// validation.
log::error(
logcat,
"Internal error: Invalid amounts passed in to save payments for address {}: "
"that address has no accrued rewards",
address_str);
return false;
}
@ -676,12 +744,14 @@ namespace cryptonote {
return true;
}
std::vector<cryptonote::batch_sn_payment> BlockchainSQLite::get_block_payments(uint64_t block_height) {
std::vector<cryptonote::batch_sn_payment> BlockchainSQLite::get_block_payments(
uint64_t block_height) {
log::trace(logcat, "BlockchainDB_SQLITE::{} Called with height: {}", __func__, block_height);
std::vector<cryptonote::batch_sn_payment> payments_at_height;
auto paid = prepared_results<std::string_view, int64_t>(
"SELECT address, amount FROM batched_payments_paid WHERE height_paid = ? ORDER BY address",
"SELECT address, amount FROM batched_payments_paid WHERE height_paid = ? ORDER BY "
"address",
static_cast<int64_t>(block_height));
for (auto [addr, amt] : paid) {

77
src/blockchain_db/sqlite/db_sqlite.h
View file

@ -27,23 +27,21 @@
#pragma once
#include <string>
#include <filesystem>
#include <SQLiteCpp/SQLiteCpp.h>
#include <filesystem>
#include <string>
#include "common/fs.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "cryptonote_core/cryptonote_tx_utils.h"
#include "sqlitedb/database.hpp"
#include "common/fs.h"
#include <SQLiteCpp/SQLiteCpp.h>
namespace cryptonote
{
namespace cryptonote {
fs::path check_if_copy_filename(std::string_view db_path);
class BlockchainSQLite : public db::Database
{
class BlockchainSQLite : public db::Database {
public:
explicit BlockchainSQLite(cryptonote::network_type nettype, fs::path db_path);
BlockchainSQLite(const BlockchainSQLite&) = delete;
@ -53,14 +51,18 @@ public:
void upgrade_schema();
void reset_database();
// The batching database maintains a height variable to know if it gets out of sync with the mainchain. Calling increment and decrement is the primary method of interacting with this height variable
// The batching database maintains a height variable to know if it gets out of sync with the
// mainchain. Calling increment and decrement is the primary method of interacting with this
// height variable
void update_height(uint64_t new_height);
void increment_height();
void decrement_height();
void blockchain_detached(uint64_t new_height);
// add_sn_payments/subtract_sn_payments -> passing an array of addresses and amounts. These will be added or subtracted to the database for each address specified. If the address does not exist it will be created.
// add_sn_payments/subtract_sn_payments -> passing an array of addresses and amounts. These will
// be added or subtracted to the database for each address specified. If the address does not
// exist it will be created.
bool add_sn_rewards(const std::vector<cryptonote::batch_sn_payment>& payments);
bool subtract_sn_rewards(const std::vector<cryptonote::batch_sn_payment>& payments);
@ -76,15 +78,16 @@ private:
std::mutex address_str_cache_mutex;
public:
// get_accrued_earnings -> queries the database for the amount that has been accrued to `service_node_address` will return the atomic value in oxen that
// the service node is owed.
// get_accrued_earnings -> queries the database for the amount that has been accrued to
// `service_node_address` will return the atomic value in oxen that the service node is owed.
uint64_t get_accrued_earnings(const std::string& address);
// get_all_accrued_earnings -> queries the database for all the amount that has been accrued to service nodes will return
// 2 vectors corresponding to the addresses and the atomic value in oxen that the service nodes are owed.
// get_all_accrued_earnings -> queries the database for all the amount that has been accrued to
// service nodes will return 2 vectors corresponding to the addresses and the atomic value in
// oxen that the service nodes are owed.
std::pair<std::vector<std::string>, std::vector<uint64_t>> get_all_accrued_earnings();
// get_payments -> passing a block height will return an array of payments that should be created in a coinbase transaction on that block given the current batching DB state.
// get_payments -> passing a block height will return an array of payments that should be
// created in a coinbase transaction on that block given the current batching DB state.
std::vector<cryptonote::batch_sn_payment> get_sn_payments(uint64_t block_height);
// calculate_rewards -> takes the list of contributors from sn_info with their SN contribution
@ -94,25 +97,37 @@ public:
//
// Note that distribution_amount here is typically passed as milli-atomic OXEN for extra
// precision.
void calculate_rewards(hf hf_version, uint64_t distribution_amount, const service_nodes::service_node_info& sn_info, std::vector<cryptonote::batch_sn_payment>& rewards);
void calculate_rewards(
hf hf_version,
uint64_t distribution_amount,
const service_nodes::service_node_info& sn_info,
std::vector<cryptonote::batch_sn_payment>& rewards);
// add/pop_block -> takes a block that contains new block rewards to be batched and added to the database
// and/or batching payments that need to be subtracted from the database, in addition it takes a reference to
// the service node state which it will use to calculate the individual payouts.
// The function will then process this block add and subtracting to the batching DB appropriately.
// This is the primary entry point for the blockchain to add to the batching database.
// Each accepted block should call this passing in the SN list structure.
bool add_block(const cryptonote::block& block, const service_nodes::service_node_list::state_t& service_nodes_state);
bool pop_block(const cryptonote::block& block, const service_nodes::service_node_list::state_t& service_nodes_state);
// add/pop_block -> takes a block that contains new block rewards to be batched and added to the
// database and/or batching payments that need to be subtracted from the database, in addition
// it takes a reference to the service node state which it will use to calculate the individual
// payouts. The function will then process this block add and subtracting to the batching DB
// appropriately. This is the primary entry point for the blockchain to add to the batching
// database. Each accepted block should call this passing in the SN list structure.
bool add_block(
const cryptonote::block& block,
const service_nodes::service_node_list::state_t& service_nodes_state);
bool pop_block(
const cryptonote::block& block,
const service_nodes::service_node_list::state_t& service_nodes_state);
// validate_batch_payment -> used to make sure that list of miner_tx_vouts is correct. Compares the miner_tx_vouts with a list previously extracted payments to make sure that the correct persons are being paid.
// validate_batch_payment -> used to make sure that list of miner_tx_vouts is correct. Compares
// the miner_tx_vouts with a list previously extracted payments to make sure that the correct
// persons are being paid.
bool validate_batch_payment(
const std::vector<std::pair<crypto::public_key, uint64_t>>& miner_tx_vouts,
const std::vector<cryptonote::batch_sn_payment>& calculated_payments_from_batching_db,
uint64_t block_height);
// these keep track of payments made to SN operators after then payment has been made. Allows for popping blocks back and knowing who got paid in those blocks.
// passing in a list of people to be marked as paid in the paid_amounts vector. Block height will be added to the batched_payments_paid database as height_paid.
// these keep track of payments made to SN operators after then payment has been made. Allows
// for popping blocks back and knowing who got paid in those blocks. passing in a list of people
// to be marked as paid in the paid_amounts vector. Block height will be added to the
// batched_payments_paid database as height_paid.
bool save_payments(uint64_t block_height, const std::vector<batch_sn_payment>& paid_amounts);
std::vector<cryptonote::batch_sn_payment> get_block_payments(uint64_t block_height);
bool delete_block_payments(uint64_t block_height);
@ -120,10 +135,8 @@ public:
uint64_t height;
protected:
cryptonote::network_type m_nettype;
std::string filename;
};
}
} // namespace cryptonote

343
src/blockchain_db/testdb.h
View file

@ -30,20 +30,23 @@
#pragma once
#include <map>
#include <string>
#include <vector>
#include <map>
#include "blockchain_db.h"
#include "cryptonote_core/service_node_list.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
namespace cryptonote
{
#include "blockchain_db.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "cryptonote_core/service_node_list.h"
namespace cryptonote {
struct checkpoint_t;
class BaseTestDB : public cryptonote::BlockchainDB {
public:
BaseTestDB() {}
virtual void open(const fs::path& filename, network_type nettype = network_type::FAKECHAIN, const int db_flags = 0) override { }
virtual void open(
const fs::path& filename,
network_type nettype = network_type::FAKECHAIN,
const int db_flags = 0) override {}
virtual void close() override {}
virtual void sync() override {}
virtual void safesyncmode(const bool onoff) override {}
@ -54,7 +57,9 @@ public:
virtual void lock() override {}
virtual bool try_lock() override { return true; }
virtual void unlock() override {}
virtual bool batch_start(uint64_t batch_num_blocks=0, uint64_t batch_bytes=0) override { return true; }
virtual bool batch_start(uint64_t batch_num_blocks = 0, uint64_t batch_bytes = 0) override {
return true;
}
virtual void batch_stop() override {}
virtual void batch_abort() override {}
virtual void set_batch_transactions(bool) override {}
@ -65,87 +70,236 @@ public:
virtual void block_rtxn_stop() const override {}
virtual void block_rtxn_abort() const override {}
virtual bool block_exists(const crypto::hash& h, uint64_t *height) const override { return false; }
virtual std::string get_block_blob_from_height(uint64_t height) const override { return cryptonote::t_serializable_object_to_blob(get_block_from_height(height)); }
virtual std::string get_block_blob(const crypto::hash& h) const override { return std::string(); }
virtual cryptonote::block_header get_block_header_from_height(uint64_t height) const override { return get_block_from_height(height); }
virtual bool get_tx_blob(const crypto::hash& h, std::string &tx) const override { return false; }
virtual bool get_pruned_tx_blob(const crypto::hash& h, std::string &tx) const override { return false; }
virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<std::string> &bd) const override { return false; }
virtual bool get_prunable_tx_blob(const crypto::hash& h, std::string &tx) const override { return false; }
virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const override { return false; }
virtual bool block_exists(const crypto::hash& h, uint64_t* height) const override {
return false;
}
virtual std::string get_block_blob_from_height(uint64_t height) const override {
return cryptonote::t_serializable_object_to_blob(get_block_from_height(height));
}
virtual std::string get_block_blob(const crypto::hash& h) const override {
return std::string();
}
virtual cryptonote::block_header get_block_header_from_height(uint64_t height) const override {
return get_block_from_height(height);
}
virtual bool get_tx_blob(const crypto::hash& h, std::string& tx) const override {
return false;
}
virtual bool get_pruned_tx_blob(const crypto::hash& h, std::string& tx) const override {
return false;
}
virtual bool get_pruned_tx_blobs_from(
const crypto::hash& h, size_t count, std::vector<std::string>& bd) const override {
return false;
}
virtual bool get_prunable_tx_blob(const crypto::hash& h, std::string& tx) const override {
return false;
}
virtual bool get_prunable_tx_hash(
const crypto::hash& tx_hash, crypto::hash& prunable_hash) const override {
return false;
}
virtual uint64_t get_block_height(const crypto::hash& h) const override { return 0; }
virtual uint64_t get_block_timestamp(const uint64_t& height) const override { return 0; }
virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const override { return {}; }
virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(
const std::vector<uint64_t>& heights) const override {
return {};
}
virtual uint64_t get_top_block_timestamp() const override { return 0; }
virtual size_t get_block_weight(const uint64_t& height) const override { return 128; }
virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const override { return {}; }
virtual cryptonote::difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const override { return 10; }
virtual cryptonote::difficulty_type get_block_difficulty(const uint64_t& height) const override { return 0; }
virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const override { return 10000000000; }
virtual uint64_t get_block_long_term_weight(const uint64_t& height) const override { return 128; }
virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const override { return {}; }
virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const override { return crypto::hash(); }
virtual std::vector<cryptonote::block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const override { return std::vector<cryptonote::block>(); }
virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const override { return std::vector<crypto::hash>(); }
virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const override { if (block_height) *block_height = 0; return crypto::hash(); }
virtual std::vector<uint64_t> get_block_weights(
uint64_t start_height, size_t count) const override {
return {};
}
virtual cryptonote::difficulty_type get_block_cumulative_difficulty(
const uint64_t& height) const override {
return 10;
}
virtual cryptonote::difficulty_type get_block_difficulty(
const uint64_t& height) const override {
return 0;
}
virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const override {
return 10000000000;
}
virtual uint64_t get_block_long_term_weight(const uint64_t& height) const override {
return 128;
}
virtual std::vector<uint64_t> get_long_term_block_weights(
uint64_t start_height, size_t count) const override {
return {};
}
virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const override {
return crypto::hash();
}
virtual std::vector<cryptonote::block> get_blocks_range(
const uint64_t& h1, const uint64_t& h2) const override {
return std::vector<cryptonote::block>();
}
virtual std::vector<crypto::hash> get_hashes_range(
const uint64_t& h1, const uint64_t& h2) const override {
return std::vector<crypto::hash>();
}
virtual crypto::hash top_block_hash(uint64_t* block_height = NULL) const override {
if (block_height)
*block_height = 0;
return crypto::hash();
}
virtual cryptonote::block get_top_block() const override { return cryptonote::block(); }
virtual uint64_t height() const override { return 1; }
virtual bool tx_exists(const crypto::hash& h) const override { return false; }
virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const override { return false; }
virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const override {
return false;
}
virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const override { return 0; }
virtual cryptonote::transaction get_tx(const crypto::hash& h) const override { return cryptonote::transaction(); }
virtual bool get_tx(const crypto::hash& h, cryptonote::transaction &tx) const override { return false; }
virtual cryptonote::transaction get_tx(const crypto::hash& h) const override {
return cryptonote::transaction();
}
virtual bool get_tx(const crypto::hash& h, cryptonote::transaction& tx) const override {
return false;
}
virtual uint64_t get_tx_count() const override { return 0; }
virtual std::vector<cryptonote::transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const override { return std::vector<cryptonote::transaction>(); }
virtual std::vector<uint64_t> get_tx_block_heights(const std::vector<crypto::hash>& h) const override { return {h.size(), 0}; }
virtual std::vector<cryptonote::transaction> get_tx_list(
const std::vector<crypto::hash>& hlist) const override {
return std::vector<cryptonote::transaction>();
}
virtual std::vector<uint64_t> get_tx_block_heights(
const std::vector<crypto::hash>& h) const override {
return {h.size(), 0};
}
virtual uint64_t get_num_outputs(const uint64_t& amount) const override { return 1; }
virtual uint64_t get_indexing_base() const override { return 0; }
virtual cryptonote::output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const override { return cryptonote::output_data_t(); }
virtual cryptonote::tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const override { return cryptonote::tx_out_index(); }
virtual cryptonote::tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const override { return cryptonote::tx_out_index(); }
virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<cryptonote::tx_out_index> &indices) const override {}
virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<cryptonote::output_data_t> &outputs, bool allow_partial = false) const override {}
virtual cryptonote::output_data_t get_output_key(
const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const override {
return cryptonote::output_data_t();
}
virtual cryptonote::tx_out_index get_output_tx_and_index_from_global(
const uint64_t& index) const override {
return cryptonote::tx_out_index();
}
virtual cryptonote::tx_out_index get_output_tx_and_index(
const uint64_t& amount, const uint64_t& index) const override {
return cryptonote::tx_out_index();
}
virtual void get_output_tx_and_index(
const uint64_t& amount,
const std::vector<uint64_t>& offsets,
std::vector<cryptonote::tx_out_index>& indices) const override {}
virtual void get_output_key(
const epee::span<const uint64_t>& amounts,
const std::vector<uint64_t>& offsets,
std::vector<cryptonote::output_data_t>& outputs,
bool allow_partial = false) const override {}
virtual bool can_thread_bulk_indices() const override { return false; }
virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_index, size_t n_txes) const override { return std::vector<std::vector<uint64_t>>(); }
virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(
const uint64_t tx_index, size_t n_txes) const override {
return std::vector<std::vector<uint64_t>>();
}
virtual bool has_key_image(const crypto::key_image& img) const override { return false; }
virtual void remove_block() override {}
virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<cryptonote::transaction, std::string>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) override {return 0;}
virtual void remove_transaction_data(const crypto::hash& tx_hash, const cryptonote::transaction& tx) override {}
virtual uint64_t add_output(const crypto::hash& tx_hash, const cryptonote::tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) override {return 0;}
virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) override {}
virtual uint64_t add_transaction_data(
const crypto::hash& blk_hash,
const std::pair<cryptonote::transaction, std::string>& tx,
const crypto::hash& tx_hash,
const crypto::hash& tx_prunable_hash) override {
return 0;
}
virtual void remove_transaction_data(
const crypto::hash& tx_hash, const cryptonote::transaction& tx) override {}
virtual uint64_t add_output(
const crypto::hash& tx_hash,
const cryptonote::tx_out& tx_output,
const uint64_t& local_index,
const uint64_t unlock_time,
const rct::key* commitment) override {
return 0;
}
virtual void add_tx_amount_output_indices(
const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) override {}
virtual void add_spent_key(const crypto::key_image& k_image) override {}
virtual void remove_spent_key(const crypto::key_image& k_image) override {}
virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const override { return true; }
virtual bool for_blocks_range(const uint64_t&, const uint64_t&, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const override { return true; }
virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const override { return true; }
virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const override { return true; }
virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const override { return true; }
virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const override {
return true;
}
virtual bool for_blocks_range(
const uint64_t&,
const uint64_t&,
std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>)
const override {
return true;
}
virtual bool for_all_transactions(
std::function<bool(const crypto::hash&, const cryptonote::transaction&)>,
bool pruned) const override {
return true;
}
virtual bool for_all_outputs(
std::function<bool(
uint64_t amount, const crypto::hash& tx_hash, uint64_t height, size_t tx_idx)>
f) const override {
return true;
}
virtual bool for_all_outputs(
uint64_t amount, const std::function<bool(uint64_t height)>& f) const override {
return true;
}
virtual bool is_read_only() const override { return false; }
virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const override { return std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>(); }
virtual bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const override { return false; }
virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(
const std::vector<uint64_t>& amounts,
bool unlocked,
uint64_t recent_cutoff,
uint64_t min_count) const override {
return std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>();
}
virtual bool get_output_distribution(
uint64_t amount,
uint64_t from_height,
uint64_t to_height,
std::vector<uint64_t>& distribution,
uint64_t& base) const override {
return false;
}
virtual void add_txpool_tx(const crypto::hash &txid, const std::string &blob, const cryptonote::txpool_tx_meta_t& details) override {}
virtual void update_txpool_tx(const crypto::hash &txid, const cryptonote::txpool_tx_meta_t& details) override {}
virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const override { return 0; }
virtual void add_txpool_tx(
const crypto::hash& txid,
const std::string& blob,
const cryptonote::txpool_tx_meta_t& details) override {}
virtual void update_txpool_tx(
const crypto::hash& txid, const cryptonote::txpool_tx_meta_t& details) override {}
virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const override {
return 0;
}
virtual bool txpool_has_tx(const crypto::hash& txid) const override { return false; }
virtual void remove_txpool_tx(const crypto::hash& txid) override {}
virtual bool get_txpool_tx_meta(const crypto::hash& txid, cryptonote::txpool_tx_meta_t &meta) const override { return false; }
virtual bool get_txpool_tx_blob(const crypto::hash& txid, std::string &bd) const override { return false; }
virtual bool get_txpool_tx_meta(
const crypto::hash& txid, cryptonote::txpool_tx_meta_t& meta) const override {
return false;
}
virtual bool get_txpool_tx_blob(const crypto::hash& txid, std::string& bd) const override {
return false;
}
virtual uint64_t get_database_size() const override { return 0; }
virtual std::string get_txpool_tx_blob(const crypto::hash& txid) const override { return ""; }
virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const std::string*)>, bool include_blob = false, bool include_unrelayed_txes = false) const override { return false; }
virtual bool for_all_txpool_txes(
std::function<bool(
const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const std::string*)>,
bool include_blob = false,
bool include_unrelayed_txes = false) const override {
return false;
}
virtual void add_block( const cryptonote::block& blk
, size_t block_weight
, uint64_t long_term_block_weight
, const cryptonote::difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, uint64_t num_rct_outs
, const crypto::hash& blk_hash
) override { }
virtual cryptonote::block get_block_from_height(uint64_t height) const override { return cryptonote::block(); }
virtual void add_block(
const cryptonote::block& blk,
size_t block_weight,
uint64_t long_term_block_weight,
const cryptonote::difficulty_type& cumulative_difficulty,
const uint64_t& coins_generated,
uint64_t num_rct_outs,
const crypto::hash& blk_hash) override {}
virtual cryptonote::block get_block_from_height(uint64_t height) const override {
return cryptonote::block();
}
virtual uint32_t get_blockchain_pruning_seed() const override { return 0; }
virtual bool prune_blockchain(uint32_t pruning_seed = 0) override { return true; }
@ -157,28 +311,65 @@ public:
virtual void add_max_block_size(uint64_t sz) override {}
virtual void update_block_checkpoint(struct checkpoint_t const& checkpoint) override {}
virtual bool get_block_checkpoint (uint64_t height, struct checkpoint_t &checkpoint) const override { return false; }
virtual bool get_top_checkpoint (struct checkpoint_t &checkpoint) const override { return false; }
virtual bool get_block_checkpoint(
uint64_t height, struct checkpoint_t& checkpoint) const override {
return false;
}
virtual bool get_top_checkpoint(struct checkpoint_t& checkpoint) const override {
return false;
}
virtual void remove_block_checkpoint(uint64_t height) override {}
std::vector<cryptonote::checkpoint_t> get_checkpoints_range(uint64_t start, uint64_t end, size_t num_desired_checkpoints = BlockchainDB::GET_ALL_CHECKPOINTS) const override { return {}; }
std::vector<cryptonote::checkpoint_t> get_checkpoints_range(
uint64_t start,
uint64_t end,
size_t num_desired_checkpoints = BlockchainDB::GET_ALL_CHECKPOINTS) const override {
return {};
}
virtual void get_output_blacklist(std::vector<uint64_t>& blacklist) const override {}
virtual void add_output_blacklist(std::vector<uint64_t> const& blacklist) override {}
virtual void set_service_node_data(const std::string& data, bool long_term) override {}
virtual bool get_service_node_data (std::string& data, bool long_term) const override { return false; }
virtual bool get_service_node_data(std::string& data, bool long_term) const override {
return false;
}
virtual void clear_service_node_data() override {}
bool get_service_node_proof(const crypto::public_key &pubkey, service_nodes::proof_info &proof) const override { return false; }
std::unordered_map<crypto::public_key, service_nodes::proof_info> get_all_service_node_proofs() const override { return {}; }
void set_service_node_proof(const crypto::public_key &pubkey, const service_nodes::proof_info &proof) override { }
bool get_service_node_proof(
const crypto::public_key& pubkey, service_nodes::proof_info& proof) const override {
return false;
}
std::unordered_map<crypto::public_key, service_nodes::proof_info> get_all_service_node_proofs()
const override {
return {};
}
void set_service_node_proof(
const crypto::public_key& pubkey, const service_nodes::proof_info& proof) override {}
bool remove_service_node_proof(const crypto::public_key& pubkey) override { return false; }
virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const std::string &blob, const std::string *checkpoint) override {}
virtual bool get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, std::string *blob, std::string *checkpoint) const override { return false; }
virtual void add_alt_block(
const crypto::hash& blkid,
const cryptonote::alt_block_data_t& data,
const std::string& blob,
const std::string* checkpoint) override {}
virtual bool get_alt_block(
const crypto::hash& blkid,
alt_block_data_t* data,
std::string* blob,
std::string* checkpoint) const override {
return false;
}
virtual void remove_alt_block(const crypto::hash& blkid) override {}
virtual uint64_t get_alt_block_count() override { return 0; }
virtual void drop_alt_blocks() override {}
virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const std::string *block_blob, const std::string *checkpoint_blob)> f, bool include_blob = false) const override { return true; }
virtual bool for_all_alt_blocks(
std::function<
bool(const crypto::hash& blkid,
const alt_block_data_t& data,
const std::string* block_blob,
const std::string* checkpoint_blob)> f,
bool include_blob = false) const override {
return true;
}
};
}
} // namespace cryptonote

475
src/blockchain_utilities/blockchain_ancestry.cpp
View file

@ -30,24 +30,25 @@
#define __STDC_FORMAT_MACROS // NOTE(oxen): Explicitly define the SCNu64 macro on Mingw
#endif
#include <cinttypes>
#include <unordered_map>
#include <unordered_set>
#include <boost/archive/portable_binary_iarchive.hpp>
#include <boost/archive/portable_binary_oarchive.hpp>
#include <cinttypes>
#include <fstream>
#include "common/unordered_containers_boost_serialization.h"
#include <unordered_map>
#include <unordered_set>
#include "blockchain_db/blockchain_db.h"
#include "blockchain_objects.h"
#include "common/command_line.h"
#include "common/varint.h"
#include "common/signal_handler.h"
#include "common/fs-format.h"
#include "serialization/boost_std_variant.h"
#include "common/signal_handler.h"
#include "common/unordered_containers_boost_serialization.h"
#include "common/varint.h"
#include "cryptonote_basic/cryptonote_boost_serialization.h"
#include "cryptonote_core/cryptonote_core.h"
#include "blockchain_objects.h"
#include "blockchain_db/blockchain_db.h"
#include "version.h"
#include "cryptonote_core/uptime_proof.h"
#include "serialization/boost_std_variant.h"
#include "version.h"
namespace po = boost::program_options;
using namespace cryptonote;
@ -55,84 +56,77 @@ using namespace cryptonote;
static auto logcat = log::Cat("bcutil");
static bool stop_requested = false;
static uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0;
static uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0,
total_blocks = 0, total_outputs = 0;
static bool opt_cache_outputs = false, opt_cache_txes = false, opt_cache_blocks = false;
struct ancestor
{
struct ancestor {
uint64_t amount;
uint64_t offset;
bool operator==(const ancestor &other) const { return amount == other.amount && offset == other.offset; }
bool operator==(const ancestor& other) const {
return amount == other.amount && offset == other.offset;
}
template <typename t_archive> void serialize(t_archive &a, const unsigned int ver)
{
template <typename t_archive>
void serialize(t_archive& a, const unsigned int ver) {
a& amount;
a& offset;
}
};
BOOST_CLASS_VERSION(ancestor, 0)
namespace std
{
template<> struct hash<ancestor>
{
size_t operator()(const ancestor &a) const
{
return a.amount ^ a.offset; // not that bad, since amount almost always have a high bit set, and offset doesn't
namespace std {
template <>
struct hash<ancestor> {
size_t operator()(const ancestor& a) const {
return a.amount ^ a.offset; // not that bad, since amount almost always have a high bit
// set, and offset doesn't
}
};
}
} // namespace std
struct tx_data_t
{
struct tx_data_t {
std::vector<std::pair<uint64_t, std::vector<uint64_t>>> vin;
std::vector<crypto::public_key> vout;
bool coinbase;
tx_data_t() : coinbase(false) {}
tx_data_t(const cryptonote::transaction &tx)
{
tx_data_t(const cryptonote::transaction& tx) {
coinbase = tx.vin.size() == 1 && std::holds_alternative<cryptonote::txin_gen>(tx.vin[0]);
if (!coinbase)
{
if (!coinbase) {
vin.reserve(tx.vin.size());
for (size_t ring = 0; ring < tx.vin.size(); ++ring)
{
for (size_t ring = 0; ring < tx.vin.size(); ++ring) {
if (const auto* txin = std::get_if<cryptonote::txin_to_key>(&tx.vin[ring]))
vin.push_back(std::make_pair(txin->amount, cryptonote::relative_output_offsets_to_absolute(txin->key_offsets)));
else
{
vin.push_back(std::make_pair(
txin->amount,
cryptonote::relative_output_offsets_to_absolute(txin->key_offsets)));
else {
log::warning(logcat, "Bad vin type in txid {}", get_transaction_hash(tx));
throw std::runtime_error("Bad vin type");
}
}
}
vout.reserve(tx.vout.size());
for (size_t out = 0; out < tx.vout.size(); ++out)
{
if (const auto* txout = std::get_if<cryptonote::txout_to_key>(&tx.vout[out].target))
{
for (size_t out = 0; out < tx.vout.size(); ++out) {
if (const auto* txout = std::get_if<cryptonote::txout_to_key>(&tx.vout[out].target)) {
vout.push_back(txout->key);
}
else
{
} else {
log::warning(logcat, "Bad vout type in txid {}", get_transaction_hash(tx));
throw std::runtime_error("Bad vout type");
}
}
}
template <typename t_archive> void serialize(t_archive &a, const unsigned int ver)
{
template <typename t_archive>
void serialize(t_archive& a, const unsigned int ver) {
a& coinbase;
a& vin;
a& vout;
}
};
struct ancestry_state_t
{
struct ancestry_state_t {
uint64_t height;
std::unordered_map<crypto::hash, std::unordered_set<ancestor>> ancestry;
std::unordered_map<ancestor, crypto::hash> output_cache;
@ -141,81 +135,80 @@ struct ancestry_state_t
ancestry_state_t() : height(0) {}
template <typename t_archive> void serialize(t_archive &a, const unsigned int ver)
{
template <typename t_archive>
void serialize(t_archive& a, const unsigned int ver) {
a& height;
a& ancestry;
a& output_cache;
if (ver < 1)
{
if (ver < 1) {
std::unordered_map<crypto::hash, cryptonote::transaction> old_tx_cache;
a& old_tx_cache;
for (const auto& [hash, tx] : old_tx_cache)
tx_cache.insert(std::make_pair(hash, ::tx_data_t(tx)));
}
else
{
} else {
a& tx_cache;
}
if (ver < 2)
{
if (ver < 2) {
std::unordered_map<uint64_t, cryptonote::block> old_block_cache;
a& old_block_cache;
block_cache.reserve(old_block_cache.size());
for (auto& [i, block] : old_block_cache)
block_cache.push_back(std::move(block));
}
else
{
} else {
a& block_cache;
}
}
};
BOOST_CLASS_VERSION(ancestry_state_t, 2)
static void add_ancestor(std::unordered_map<ancestor, unsigned int> &ancestry, uint64_t amount, uint64_t offset)
{
std::pair<std::unordered_map<ancestor, unsigned int>::iterator, bool> p = ancestry.insert(std::make_pair(ancestor{amount, offset}, 1));
if (!p.second)
{
static void add_ancestor(
std::unordered_map<ancestor, unsigned int>& ancestry, uint64_t amount, uint64_t offset) {
std::pair<std::unordered_map<ancestor, unsigned int>::iterator, bool> p =
ancestry.insert(std::make_pair(ancestor{amount, offset}, 1));
if (!p.second) {
++p.first->second;
}
}
static size_t get_full_ancestry(const std::unordered_map<ancestor, unsigned int> &ancestry)
{
static size_t get_full_ancestry(const std::unordered_map<ancestor, unsigned int>& ancestry) {
size_t count = 0;
for (const auto& i : ancestry)
count += i.second;
return count;
}
static size_t get_deduplicated_ancestry(const std::unordered_map<ancestor, unsigned int> &ancestry)
{
static size_t get_deduplicated_ancestry(
const std::unordered_map<ancestor, unsigned int>& ancestry) {
return ancestry.size();
}
static void add_ancestry(std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid, const std::unordered_set<ancestor> &ancestors)
{
std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p = ancestry.insert(std::make_pair(txid, ancestors));
if (!p.second)
{
static void add_ancestry(
std::unordered_map<crypto::hash, std::unordered_set<ancestor>>& ancestry,
const crypto::hash& txid,
const std::unordered_set<ancestor>& ancestors) {
std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p =
ancestry.insert(std::make_pair(txid, ancestors));
if (!p.second) {
for (const auto& e : ancestors)
p.first->second.insert(e);
}
}
static void add_ancestry(std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid, const ancestor &new_ancestor)
{
std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p = ancestry.insert(std::make_pair(txid, std::unordered_set<ancestor>()));
static void add_ancestry(
std::unordered_map<crypto::hash, std::unordered_set<ancestor>>& ancestry,
const crypto::hash& txid,
const ancestor& new_ancestor) {
std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p =
ancestry.insert(std::make_pair(txid, std::unordered_set<ancestor>()));
p.first->second.insert(new_ancestor);
}
static std::unordered_set<ancestor> get_ancestry(const std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid)
{
std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::const_iterator i = ancestry.find(txid);
if (i == ancestry.end())
{
static std::unordered_set<ancestor> get_ancestry(
const std::unordered_map<crypto::hash, std::unordered_set<ancestor>>& ancestry,
const crypto::hash& txid) {
std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::const_iterator i =
ancestry.find(txid);
if (i == ancestry.end()) {
// log::error(logcat, "txid ancestry not found: {}", txid);
// throw std::runtime_error("txid ancestry not found");
return std::unordered_set<ancestor>();
@ -223,49 +216,43 @@ static std::unordered_set<ancestor> get_ancestry(const std::unordered_map<crypto
return i->second;
}
static bool get_block_from_height(ancestry_state_t &state, BlockchainDB *db, uint64_t height, cryptonote::block &b)
{
static bool get_block_from_height(
ancestry_state_t& state, BlockchainDB* db, uint64_t height, cryptonote::block& b) {
++total_blocks;
if (state.block_cache.size() > height && !state.block_cache[height].miner_tx.vin.empty())
{
if (state.block_cache.size() > height && !state.block_cache[height].miner_tx.vin.empty()) {
++cached_blocks;
b = state.block_cache[height];
return true;
}
std::string bd = db->get_block_blob_from_height(height);
if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
{
if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) {
log::warning(logcat, "Bad block from db");
return false;
}
if (opt_cache_blocks)
{
if (opt_cache_blocks) {
state.block_cache.resize(height + 1);
state.block_cache[height] = b;
}
return true;
}
static bool get_transaction(ancestry_state_t &state, BlockchainDB *db, const crypto::hash &txid, ::tx_data_t &tx_data)
{
static bool get_transaction(
ancestry_state_t& state, BlockchainDB* db, const crypto::hash& txid, ::tx_data_t& tx_data) {
std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(txid);
++total_txes;
if (i != state.tx_cache.end())
{
if (i != state.tx_cache.end()) {
++cached_txes;
tx_data = i->second;
return true;
}
std::string bd;
if (!db->get_pruned_tx_blob(txid, bd))
{
if (!db->get_pruned_tx_blob(txid, bd)) {
log::warning(logcat, "Failed to get txid {} from db", txid);
return false;
}
cryptonote::transaction tx;
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx))
{
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) {
log::warning(logcat, "Bad tx: {}", txid);
return false;
}
@ -275,12 +262,16 @@ static bool get_transaction(ancestry_state_t &state, BlockchainDB *db, const cry
return true;
}
static bool get_output_txid(ancestry_state_t &state, BlockchainDB *db, uint64_t amount, uint64_t offset, crypto::hash &txid)
{
static bool get_output_txid(
ancestry_state_t& state,
BlockchainDB* db,
uint64_t amount,
uint64_t offset,
crypto::hash& txid) {
++total_outputs;
std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset});
if (i != state.output_cache.end())
{
std::unordered_map<ancestor, crypto::hash>::const_iterator i =
state.output_cache.find({amount, offset});
if (i != state.output_cache.end()) {
++cached_outputs;
txid = i->second;
return true;
@ -291,34 +282,30 @@ static bool get_output_txid(ancestry_state_t &state, BlockchainDB *db, uint64_t
if (!get_block_from_height(state, db, od.height, b))
return false;
for (size_t out = 0; out < b.miner_tx.vout.size(); ++out)
{
if (const auto* txout = std::get_if<cryptonote::txout_to_key>(&b.miner_tx.vout[out].target))
{
if (txout->key == od.pubkey)
{
for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) {
if (const auto* txout =
std::get_if<cryptonote::txout_to_key>(&b.miner_tx.vout[out].target)) {
if (txout->key == od.pubkey) {
txid = cryptonote::get_transaction_hash(b.miner_tx);
if (opt_cache_outputs)
state.output_cache.insert(std::make_pair(ancestor{amount, offset}, txid));
return true;
}
}
else
{
log::warning(logcat, "Bad vout type in txid {}", cryptonote::get_transaction_hash(b.miner_tx));
} else {
log::warning(
logcat,
"Bad vout type in txid {}",
cryptonote::get_transaction_hash(b.miner_tx));
return false;
}
}
for (const crypto::hash &block_txid: b.tx_hashes)
{
for (const crypto::hash& block_txid : b.tx_hashes) {
::tx_data_t tx_data3;
if (!get_transaction(state, db, block_txid, tx_data3))
return false;
for (size_t out = 0; out < tx_data3.vout.size(); ++out)
{
if (tx_data3.vout[out] == od.pubkey)
{
for (size_t out = 0; out < tx_data3.vout.size(); ++out) {
if (tx_data3.vout[out] == od.pubkey) {
txid = block_txid;
if (opt_cache_outputs)
state.output_cache.insert(std::make_pair(ancestor{amount, offset}, txid));
@ -329,8 +316,7 @@ static bool get_output_txid(ancestry_state_t &state, BlockchainDB *db, uint64_t
return false;
}
int main(int argc, char* argv[])
{
int main(int argc, char* argv[]) {
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
@ -340,17 +326,28 @@ int main(int argc, char* argv[])
auto opt_size = command_line::boost_option_sizes();
po::options_description desc_cmd_only("Command line options", opt_size.first, opt_size.second);
po::options_description desc_cmd_sett("Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get ancestry for this txid", ""};
const command_line::arg_descriptor<std::string> arg_output = {"output", "Get ancestry for this output (amount/offset format)", ""};
const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get ancestry for all txes at this height", 0};
const command_line::arg_descriptor<bool> arg_refresh = {"refresh", "Refresh the whole chain first", false};
const command_line::arg_descriptor<bool> arg_cache_outputs = {"cache-outputs", "Cache outputs (memory hungry)", false};
const command_line::arg_descriptor<bool> arg_cache_txes = {"cache-txes", "Cache txes (memory hungry)", false};
const command_line::arg_descriptor<bool> arg_cache_blocks = {"cache-blocks", "Cache blocks (memory hungry)", false};
const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Including coinbase tx in per height average", false};
const command_line::arg_descriptor<bool> arg_show_cache_stats = {"show-cache-stats", "Show cache statistics", false};
po::options_description desc_cmd_sett(
"Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_log_level = {
"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<std::string> arg_txid = {
"txid", "Get ancestry for this txid", ""};
const command_line::arg_descriptor<std::string> arg_output = {
"output", "Get ancestry for this output (amount/offset format)", ""};
const command_line::arg_descriptor<uint64_t> arg_height = {
"height", "Get ancestry for all txes at this height", 0};
const command_line::arg_descriptor<bool> arg_refresh = {
"refresh", "Refresh the whole chain first", false};
const command_line::arg_descriptor<bool> arg_cache_outputs = {
"cache-outputs", "Cache outputs (memory hungry)", false};
const command_line::arg_descriptor<bool> arg_cache_txes = {
"cache-txes", "Cache txes (memory hungry)", false};
const command_line::arg_descriptor<bool> arg_cache_blocks = {
"cache-blocks", "Cache blocks (memory hungry)", false};
const command_line::arg_descriptor<bool> arg_include_coinbase = {
"include-coinbase", "Including coinbase tx in per height average", false};
const command_line::arg_descriptor<bool> arg_show_cache_stats = {
"show-cache-stats", "Show cache statistics", false};
command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
@ -371,8 +368,7 @@ int main(int argc, char* argv[])
desc_options.add(desc_cmd_only).add(desc_cmd_sett);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
{
bool r = command_line::handle_error_helper(desc_options, [&]() {
auto parser = po::command_line_parser(argc, argv).options(desc_options);
po::store(parser.run(), vm);
po::notify(vm);
@ -381,8 +377,7 @@ int main(int argc, char* argv[])
if (!r)
return 1;
if (command_line::get_arg(vm, command_line::arg_help))
{
if (command_line::get_arg(vm, command_line::arg_help)) {
std::cout << "Oxen '" << OXEN_RELEASE_NAME << "' (v" << OXEN_VERSION_FULL << ")\n\n";
std::cout << desc_options << std::endl;
return 1;
@ -393,7 +388,8 @@ int main(int argc, char* argv[])
if (auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str()
<< std::endl;
throw std::runtime_error{"Incorrect log level"};
}
oxen::logging::init(log_file_path, log_level);
@ -402,7 +398,9 @@ int main(int argc, char* argv[])
std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir);
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
network_type net_type = opt_testnet ? network_type::TESTNET : opt_devnet ? network_type::DEVNET : network_type::MAINNET;
network_type net_type = opt_testnet ? network_type::TESTNET
: opt_devnet ? network_type::DEVNET
: network_type::MAINNET;
std::string opt_txid_string = command_line::get_arg(vm, arg_txid);
std::string opt_output_string = command_line::get_arg(vm, arg_output);
uint64_t opt_height = command_line::get_arg(vm, arg_height);
@ -413,25 +411,22 @@ int main(int argc, char* argv[])
bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase);
bool opt_show_cache_stats = command_line::get_arg(vm, arg_show_cache_stats);
if ((!opt_txid_string.empty()) + !!opt_height + !opt_output_string.empty() > 1)
{
if ((!opt_txid_string.empty()) + !!opt_height + !opt_output_string.empty() > 1) {
std::cerr << "Only one of --txid, --height, --output can be given" << std::endl;
return 1;
}
crypto::hash opt_txid{};
uint64_t output_amount = 0, output_offset = 0;
if (!opt_txid_string.empty())
{
if (!tools::hex_to_type(opt_txid_string, opt_txid))
{
if (!opt_txid_string.empty()) {
if (!tools::hex_to_type(opt_txid_string, opt_txid)) {
std::cerr << "Invalid txid" << std::endl;
return 1;
}
}
else if (!opt_output_string.empty())
{
if (sscanf(opt_output_string.c_str(), "%" SCNu64 "/%" SCNu64, &output_amount, &output_offset) != 2)
{
} else if (!opt_output_string.empty()) {
if (sscanf(opt_output_string.c_str(),
"%" SCNu64 "/%" SCNu64,
&output_amount,
&output_offset) != 2) {
std::cerr << "Invalid output" << std::endl;
return 1;
}
@ -441,8 +436,7 @@ int main(int argc, char* argv[])
blockchain_objects_t blockchain_objects = {};
Blockchain* core_storage = &blockchain_objects.m_blockchain;
BlockchainDB* db = new_db();
if (db == NULL)
{
if (db == NULL) {
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
@ -451,12 +445,9 @@ int main(int argc, char* argv[])
fs::path filename = fs::u8path(opt_data_dir) / db->get_db_name();
log::warning(logcat, "Loading blockchain from folder {} ...", filename);
try
{
try {
db->open(filename, core_storage->nettype(), DBF_RDONLY);
}
catch (const std::exception& e)
{
} catch (const std::exception& e) {
log::warning(logcat, "Error opening database: {}", e.what());
return 1;
}
@ -473,44 +464,37 @@ int main(int argc, char* argv[])
log::warning(logcat, "Loading state data from {}", state_file_path);
fs::ifstream state_data_in;
state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in);
if (!state_data_in.fail())
{
try
{
if (!state_data_in.fail()) {
try {
boost::archive::portable_binary_iarchive a(state_data_in);
a >> state;
}
catch (const std::exception &e)
{
log::error(logcat, "Failed to load state data from {}, restarting from scratch", state_file_path);
} catch (const std::exception& e) {
log::error(
logcat,
"Failed to load state data from {}, restarting from scratch",
state_file_path);
state = ancestry_state_t();
}
state_data_in.close();
}
tools::signal_handler::install([](int type) {
stop_requested = true;
});
tools::signal_handler::install([](int type) { stop_requested = true; });
// forward method
const uint64_t db_height = db->height();
if (opt_refresh)
{
if (opt_refresh) {
log::info(logcat, "Starting from height {}", state.height);
state.block_cache.reserve(db_height);
for (uint64_t h = state.height; h < db_height; ++h)
{
for (uint64_t h = state.height; h < db_height; ++h) {
size_t block_ancestry_size = 0;
const std::string bd = db->get_block_blob_from_height(h);
++total_blocks;
cryptonote::block b;
if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
{
if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) {
log::warning(logcat, "Bad block from db");
return 1;
}
if (opt_cache_blocks)
{
if (opt_cache_blocks) {
state.block_cache.resize(h + 1);
state.block_cache[h] = b;
}
@ -520,29 +504,24 @@ int main(int argc, char* argv[])
txids.push_back(cryptonote::get_transaction_hash(b.miner_tx));
for (const auto& h : b.tx_hashes)
txids.push_back(h);
for (const crypto::hash &txid: txids)
{
for (const crypto::hash& txid : txids) {
printf("%lu/%lu \r", (unsigned long)h, (unsigned long)db_height);
fflush(stdout);
::tx_data_t tx_data;
std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(txid);
std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i =
state.tx_cache.find(txid);
++total_txes;
if (i != state.tx_cache.end())
{
if (i != state.tx_cache.end()) {
++cached_txes;
tx_data = i->second;
}
else
{
} else {
std::string bd;
if (!db->get_pruned_tx_blob(txid, bd))
{
if (!db->get_pruned_tx_blob(txid, bd)) {
log::warning(logcat, "Failed to get txid {} from db", txid);
return 1;
}
cryptonote::transaction tx;
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx))
{
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) {
log::warning(logcat, "Bad tx: {}", txid);
return 1;
}
@ -550,28 +529,25 @@ int main(int argc, char* argv[])
if (opt_cache_txes)
state.tx_cache.insert(std::make_pair(txid, tx_data));
}
if (tx_data.coinbase)
{
if (tx_data.coinbase) {
add_ancestry(state.ancestry, txid, std::unordered_set<ancestor>());
}
else
{
for (size_t ring = 0; ring < tx_data.vin.size(); ++ring)
{
} else {
for (size_t ring = 0; ring < tx_data.vin.size(); ++ring) {
const uint64_t amount = tx_data.vin[ring].first;
const std::vector<uint64_t>& absolute_offsets = tx_data.vin[ring].second;
for (uint64_t offset: absolute_offsets)
{
for (uint64_t offset : absolute_offsets) {
add_ancestry(state.ancestry, txid, ancestor{amount, offset});
// find the tx which created this output
bool found = false;
crypto::hash output_txid;
if (!get_output_txid(state, db, amount, offset, output_txid))
{
if (!get_output_txid(state, db, amount, offset, output_txid)) {
log::warning(logcat, "Output originating transaction not found");
return 1;
}
add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid));
add_ancestry(
state.ancestry,
txid,
get_ancestry(state.ancestry, output_txid));
}
}
}
@ -579,10 +555,15 @@ int main(int argc, char* argv[])
block_ancestry_size += ancestry_size;
log::info(logcat, "{}: {}", txid, ancestry_size);
}
if (!txids.empty())
{
if (!txids.empty()) {
std::string stats_msg;
log::info(logcat, "Height {}: {} average over {}{}", h, (block_ancestry_size / txids.size()), txids.size(), stats_msg);
log::info(
logcat,
"Height {}: {} average over {}{}",
h,
(block_ancestry_size / txids.size()),
txids.size(),
stats_msg);
}
state.height = h;
if (stop_requested)
@ -591,50 +572,45 @@ int main(int argc, char* argv[])
log::warning(logcat, "Saving state data to {}", state_file_path);
std::ofstream state_data_out;
state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc);
if (!state_data_out.fail())
{
try
{
state_data_out.open(
state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc);
if (!state_data_out.fail()) {
try {
boost::archive::portable_binary_oarchive a(state_data_out);
a << state;
}
catch (const std::exception &e)
{
} catch (const std::exception& e) {
log::error(logcat, "Failed to save state data to {}", state_file_path);
}
state_data_out.close();
}
}
else
{
if (state.height < db_height)
{
log::warning(logcat, "The state file is only built up to height {}, but the blockchain reached height {}", state.height, db_height);
log::warning(logcat, "You may want to run with --refresh if you want to get ancestry for newer data");
} else {
if (state.height < db_height) {
log::warning(
logcat,
"The state file is only built up to height {}, but the blockchain reached "
"height {}",
state.height,
db_height);
log::warning(
logcat,
"You may want to run with --refresh if you want to get ancestry for newer "
"data");
}
}
if (!opt_txid_string.empty())
{
if (!opt_txid_string.empty()) {
start_txids.push_back(opt_txid);
}
else if (!opt_output_string.empty())
{
} else if (!opt_output_string.empty()) {
crypto::hash txid;
if (!get_output_txid(state, db, output_amount, output_offset, txid))
{
if (!get_output_txid(state, db, output_amount, output_offset, txid)) {
log::warning(logcat, "Output not found in db");
return 1;
}
start_txids.push_back(txid);
}
else
{
} else {
const std::string bd = db->get_block_blob_from_height(opt_height);
cryptonote::block b;
if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
{
if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) {
log::warning(logcat, "Bad block from db");
return 1;
}
@ -642,22 +618,19 @@ int main(int argc, char* argv[])
start_txids.push_back(txid);
}
if (start_txids.empty())
{
if (start_txids.empty()) {
log::warning(logcat, "No transaction(s) to check");
return 1;
}
for (const crypto::hash &start_txid: start_txids)
{
for (const crypto::hash& start_txid : start_txids) {
log::warning(logcat, "Checking ancestry for txid {}", start_txid);
std::unordered_map<ancestor, unsigned int> ancestry;
std::list<crypto::hash> txids;
txids.push_back(start_txid);
while (!txids.empty())
{
while (!txids.empty()) {
const crypto::hash txid = txids.front();
txids.pop_front();
@ -672,25 +645,23 @@ int main(int argc, char* argv[])
if (coinbase)
continue;
for (size_t ring = 0; ring < tx_data2.vin.size(); ++ring)
{
for (size_t ring = 0; ring < tx_data2.vin.size(); ++ring) {
{
const uint64_t amount = tx_data2.vin[ring].first;
auto absolute_offsets = tx_data2.vin[ring].second;
for (uint64_t offset: absolute_offsets)
{
for (uint64_t offset : absolute_offsets) {
add_ancestor(ancestry, amount, offset);
// find the tx which created this output
bool found = false;
crypto::hash output_txid;
if (!get_output_txid(state, db, amount, offset, output_txid))
{
if (!get_output_txid(state, db, amount, offset, output_txid)) {
log::warning(logcat, "Output originating transaction not found");
return 1;
}
add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid));
add_ancestry(
state.ancestry, txid, get_ancestry(state.ancestry, output_txid));
txids.push_back(output_txid);
log::debug(logcat, "adding txid: {}", output_txid);
}
@ -698,10 +669,19 @@ int main(int argc, char* argv[])
}
}
log::info(logcat, "Ancestry for {}: {} / {}", start_txid, get_deduplicated_ancestry(ancestry), get_full_ancestry(ancestry));
for (const auto &i: ancestry)
{
log::info(logcat, "{}/{}: {}", cryptonote::print_money(i.first.amount), i.first.offset, i.second);
log::info(
logcat,
"Ancestry for {}: {} / {}",
start_txid,
get_deduplicated_ancestry(ancestry),
get_full_ancestry(ancestry));
for (const auto& i : ancestry) {
log::info(
logcat,
"{}/{}: {}",
cryptonote::print_money(i.first.amount),
i.first.offset,
i.second);
}
}
@ -709,7 +689,12 @@ done:
core_storage->deinit();
if (opt_show_cache_stats)
log::info(logcat, "cache: txes {}%, blocks {}%, outputs {}%", std::to_string(cached_txes*100./total_txes), std::to_string(cached_blocks*100./total_blocks), std::to_string(cached_outputs*100./total_outputs));
log::info(
logcat,
"cache: txes {}%, blocks {}%, outputs {}%",
std::to_string(cached_txes * 100. / total_txes),
std::to_string(cached_blocks * 100. / total_blocks),
std::to_string(cached_outputs * 100. / total_outputs));
return 0;

1211
src/blockchain_utilities/blockchain_blackball.cpp
View file

File diff suppressed because it is too large Load diff

192
src/blockchain_utilities/blockchain_depth.cpp
View file

@ -26,13 +26,13 @@
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "common/command_line.h"
#include "common/varint.h"
#include "common/median.h"
#include "common/fs-format.h"
#include "cryptonote_core/cryptonote_core.h"
#include "blockchain_objects.h"
#include "blockchain_db/blockchain_db.h"
#include "blockchain_objects.h"
#include "common/command_line.h"
#include "common/fs-format.h"
#include "common/median.h"
#include "common/varint.h"
#include "cryptonote_core/cryptonote_core.h"
#include "cryptonote_core/uptime_proof.h"
#include "version.h"
@ -41,8 +41,7 @@ using namespace cryptonote;
static auto logcat = log::Cat("bcutil");
int main(int argc, char* argv[])
{
int main(int argc, char* argv[]) {
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
@ -51,11 +50,16 @@ int main(int argc, char* argv[])
auto opt_size = command_line::boost_option_sizes();
po::options_description desc_cmd_only("Command line options", opt_size.first, opt_size.second);
po::options_description desc_cmd_sett("Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get min depth for this txid", ""};
const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get min depth for all txes at this height", 0};
const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Include coinbase in the average", false};
po::options_description desc_cmd_sett(
"Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_log_level = {
"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<std::string> arg_txid = {
"txid", "Get min depth for this txid", ""};
const command_line::arg_descriptor<uint64_t> arg_height = {
"height", "Get min depth for all txes at this height", 0};
const command_line::arg_descriptor<bool> arg_include_coinbase = {
"include-coinbase", "Include coinbase in the average", false};
command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
@ -70,8 +74,7 @@ int main(int argc, char* argv[])
desc_options.add(desc_cmd_only).add(desc_cmd_sett);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
{
bool r = command_line::handle_error_helper(desc_options, [&]() {
auto parser = po::command_line_parser(argc, argv).options(desc_options);
po::store(parser.run(), vm);
po::notify(vm);
@ -80,8 +83,7 @@ int main(int argc, char* argv[])
if (!r)
return 1;
if (command_line::get_arg(vm, command_line::arg_help))
{
if (command_line::get_arg(vm, command_line::arg_help)) {
std::cout << "Oxen '" << OXEN_RELEASE_NAME << "' (v" << OXEN_VERSION_FULL << ")\n\n";
std::cout << desc_options << std::endl;
return 1;
@ -93,7 +95,8 @@ int main(int argc, char* argv[])
if (auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str()
<< std::endl;
throw std::runtime_error{"Incorrect log level"};
}
oxen::logging::init(log_file_path, log_level);
@ -101,21 +104,20 @@ int main(int argc, char* argv[])
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
network_type net_type = opt_testnet ? network_type::TESTNET : opt_devnet ? network_type::DEVNET : network_type::MAINNET;
network_type net_type = opt_testnet ? network_type::TESTNET
: opt_devnet ? network_type::DEVNET
: network_type::MAINNET;
std::string opt_txid_string = command_line::get_arg(vm, arg_txid);
uint64_t opt_height = command_line::get_arg(vm, arg_height);
bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase);
if (!opt_txid_string.empty() && opt_height)
{
if (!opt_txid_string.empty() && opt_height) {
std::cerr << "txid and height cannot be given at the same time" << std::endl;
return 1;
}
crypto::hash opt_txid{};
if (!opt_txid_string.empty())
{
if (!tools::hex_to_type(opt_txid_string, opt_txid))
{
if (!opt_txid_string.empty()) {
if (!tools::hex_to_type(opt_txid_string, opt_txid)) {
std::cerr << "Invalid txid" << std::endl;
return 1;
}
@ -125,22 +127,19 @@ int main(int argc, char* argv[])
blockchain_objects_t blockchain_objects = {};
Blockchain* core_storage = &blockchain_objects.m_blockchain;
BlockchainDB* db = new_db();
if (db == NULL)
{
if (db == NULL) {
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
log::warning(logcat, "database: LMDB");
const fs::path filename = fs::u8path(command_line::get_arg(vm, cryptonote::arg_data_dir)) / db->get_db_name();
const fs::path filename =
fs::u8path(command_line::get_arg(vm, cryptonote::arg_data_dir)) / db->get_db_name();
log::warning(logcat, "Loading blockchain from folder {} ...", filename);
try
{
try {
db->open(filename, core_storage->nettype(), DBF_RDONLY);
}
catch (const std::exception& e)
{
} catch (const std::exception& e) {
log::warning(logcat, "Error opening database: {}", e.what());
return 1;
}
@ -150,16 +149,12 @@ int main(int argc, char* argv[])
log::warning(logcat, "Source blockchain storage initialized OK");
std::vector<crypto::hash> start_txids;
if (!opt_txid_string.empty())
{
if (!opt_txid_string.empty()) {
start_txids.push_back(opt_txid);
}
else
{
} else {
const std::string bd = db->get_block_blob_from_height(opt_height);
cryptonote::block b;
if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
{
if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) {
log::warning(logcat, "Bad block from db");
return 1;
}
@ -169,131 +164,116 @@ int main(int argc, char* argv[])
start_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx));
}
if (start_txids.empty())
{
if (start_txids.empty()) {
log::warning(logcat, "No transaction(s) to check");
return 1;
}
std::vector<uint64_t> depths;
for (const crypto::hash &start_txid: start_txids)
{
for (const crypto::hash& start_txid : start_txids) {
uint64_t depth = 0;
bool coinbase = false;
log::warning(logcat, "Checking depth for txid {}", start_txid);
std::vector<crypto::hash> txids(1, start_txid);
while (!coinbase)
{
while (!coinbase) {
log::warning(logcat, "Considering {} transaction(s) at depth {}", txids.size(), depth);
std::vector<crypto::hash> new_txids;
for (const crypto::hash &txid: txids)
{
for (const crypto::hash& txid : txids) {
std::string bd;
if (!db->get_pruned_tx_blob(txid, bd))
{
if (!db->get_pruned_tx_blob(txid, bd)) {
log::warning(logcat, "Failed to get txid {} from db", txid);
return 1;
}
cryptonote::transaction tx;
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx))
{
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) {
log::warning(logcat, "Bad tx: {}", txid);
return 1;
}
for (size_t ring = 0; ring < tx.vin.size(); ++ring)
{
if (std::holds_alternative<cryptonote::txin_gen>(tx.vin[ring]))
{
for (size_t ring = 0; ring < tx.vin.size(); ++ring) {
if (std::holds_alternative<cryptonote::txin_gen>(tx.vin[ring])) {
log::debug(logcat, "{} is a coinbase transaction", txid);
coinbase = true;
goto done;
}
if (auto* txin = std::get_if<cryptonote::txin_to_key>(&tx.vin[ring]))
{
if (auto* txin = std::get_if<cryptonote::txin_to_key>(&tx.vin[ring])) {
const uint64_t amount = txin->amount;
auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin->key_offsets);
for (uint64_t offset: absolute_offsets)
{
auto absolute_offsets =
cryptonote::relative_output_offsets_to_absolute(txin->key_offsets);
for (uint64_t offset : absolute_offsets) {
const output_data_t od = db->get_output_key(amount, offset);
const crypto::hash block_hash = db->get_block_hash_from_height(od.height);
const crypto::hash block_hash =
db->get_block_hash_from_height(od.height);
bd = db->get_block_blob(block_hash);
cryptonote::block b;
if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
{
if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) {
log::warning(logcat, "Bad block from db");
return 1;
}
// find the tx which created this output
bool found = false;
for (size_t out = 0; out < b.miner_tx.vout.size(); ++out)
{
if (auto* txout = std::get_if<cryptonote::txout_to_key>(&b.miner_tx.vout[out].target))
{
if (txout->key == od.pubkey)
{
for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) {
if (auto* txout = std::get_if<cryptonote::txout_to_key>(
&b.miner_tx.vout[out].target)) {
if (txout->key == od.pubkey) {
found = true;
new_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx));
log::debug(logcat, "adding txid: {}", cryptonote::get_transaction_hash(b.miner_tx));
new_txids.push_back(
cryptonote::get_transaction_hash(b.miner_tx));
log::debug(
logcat,
"adding txid: {}",
cryptonote::get_transaction_hash(b.miner_tx));
break;
}
}
else
{
log::warning(logcat, "Bad vout type in txid {}", cryptonote::get_transaction_hash(b.miner_tx));
} else {
log::warning(
logcat,
"Bad vout type in txid {}",
cryptonote::get_transaction_hash(b.miner_tx));
return 1;
}
}
for (const crypto::hash &block_txid: b.tx_hashes)
{
for (const crypto::hash& block_txid : b.tx_hashes) {
if (found)
break;
if (!db->get_pruned_tx_blob(block_txid, bd))
{
log::warning(logcat, "Failed to get txid {} from db", block_txid);
if (!db->get_pruned_tx_blob(block_txid, bd)) {
log::warning(
logcat, "Failed to get txid {} from db", block_txid);
return 1;
}
cryptonote::transaction tx2;
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2))
{
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) {
log::warning(logcat, "Bad tx: {}", block_txid);
return 1;
}
for (size_t out = 0; out < tx2.vout.size(); ++out)
{
if (auto* txout = std::get_if<cryptonote::txout_to_key>(&tx2.vout[out].target))
{
if (txout->key == od.pubkey)
{
for (size_t out = 0; out < tx2.vout.size(); ++out) {
if (auto* txout = std::get_if<cryptonote::txout_to_key>(
&tx2.vout[out].target)) {
if (txout->key == od.pubkey) {
found = true;
new_txids.push_back(block_txid);
log::debug(logcat, "adding txid: {}", block_txid);
break;
}
}
else
{
log::warning(logcat, "Bad vout type in txid {}", block_txid);
} else {
log::warning(
logcat, "Bad vout type in txid {}", block_txid);
return 1;
}
}
}
if (!found)
{
if (!found) {
log::warning(logcat, "Output originating transaction not found");
return 1;
}
}
}
else
{
} else {
log::warning(logcat, "Bad vin type in txid {}", txid);
return 1;
}
}
}
if (!coinbase)
{
if (!coinbase) {
std::swap(txids, new_txids);
++depth;
}
@ -306,8 +286,16 @@ done:
uint64_t cumulative_depth = 0;
for (uint64_t depth : depths)
cumulative_depth += depth;
log::warning(logcat, "Average min depth for {} transaction(s): {}", start_txids.size(), cumulative_depth/(float)depths.size());
log::warning(logcat, "Median min depth for {} transaction(s): {}", start_txids.size(), tools::median(std::move(depths)));
log::warning(
logcat,
"Average min depth for {} transaction(s): {}",
start_txids.size(),
cumulative_depth / (float)depths.size());
log::warning(
logcat,
"Median min depth for {} transaction(s): {}",
start_txids.size(),
tools::median(std::move(depths)));
core_storage->deinit();
return 0;

65
src/blockchain_utilities/blockchain_export.cpp
View file

@ -27,19 +27,18 @@
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "bootstrap_file.h"
#include "blockchain_objects.h"
#include "blocksdat_file.h"
#include "bootstrap_file.h"
#include "common/command_line.h"
#include "common/fs-format.h"
#include "cryptonote_core/cryptonote_core.h"
#include "blockchain_objects.h"
#include "version.h"
#include "cryptonote_core/uptime_proof.h"
#include "version.h"
namespace po = boost::program_options;
int main(int argc, char* argv[])
{
int main(int argc, char* argv[]) {
using namespace oxen;
auto logcat = log::Cat("bcutil");
@ -52,12 +51,16 @@ int main(int argc, char* argv[])
auto opt_size = command_line::boost_option_sizes();
po::options_description desc_cmd_only("Command line options", opt_size.first, opt_size.second);
po::options_description desc_cmd_sett("Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_output_file = {"output-file", "Specify output file", "", true};
const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop};
const command_line::arg_descriptor<bool> arg_blocks_dat = {"blocksdat", "Output in blocks.dat format", blocks_dat};
po::options_description desc_cmd_sett(
"Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_output_file = {
"output-file", "Specify output file", "", true};
const command_line::arg_descriptor<std::string> arg_log_level = {
"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<uint64_t> arg_block_stop = {
"block-stop", "Stop at block number", block_stop};
const command_line::arg_descriptor<bool> arg_blocks_dat = {
"blocksdat", "Output in blocks.dat format", blocks_dat};
command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
command_line::add_arg(desc_cmd_sett, arg_output_file);
@ -73,8 +76,7 @@ int main(int argc, char* argv[])
desc_options.add(desc_cmd_only).add(desc_cmd_sett);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
{
bool r = command_line::handle_error_helper(desc_options, [&]() {
po::store(po::parse_command_line(argc, argv, desc_options), vm);
po::notify(vm);
return true;
@ -82,8 +84,7 @@ int main(int argc, char* argv[])
if (!r)
return 1;
if (command_line::get_arg(vm, command_line::arg_help))
{
if (command_line::get_arg(vm, command_line::arg_help)) {
std::cout << "Oxen '" << OXEN_RELEASE_NAME << "' (v" << OXEN_VERSION_FULL << ")\n\n";
std::cout << desc_options << std::endl;
return 1;
@ -97,7 +98,8 @@ int main(int argc, char* argv[])
if (auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str()
<< std::endl;
throw std::runtime_error{"Incorrect log level"};
}
oxen::logging::init(log_file_path, log_level);
@ -105,8 +107,7 @@ int main(int argc, char* argv[])
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
if (opt_testnet && opt_devnet)
{
if (opt_testnet && opt_devnet) {
std::cerr << "Can't specify more than one of --testnet and --devnet" << std::endl;
return 1;
}
@ -125,8 +126,7 @@ int main(int argc, char* argv[])
blockchain_objects_t blockchain_objects = {};
Blockchain* core_storage = &blockchain_objects.m_blockchain;
BlockchainDB* db = new_db();
if (db == NULL)
{
if (db == NULL) {
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
@ -135,19 +135,21 @@ int main(int argc, char* argv[])
auto filename = config_folder / db->get_db_name();
log::warning(logcat, "Loading blockchain from folder {} ...", filename);
try
{
try {
db->open(filename, core_storage->nettype(), DBF_RDONLY);
}
catch (const std::exception& e)
{
} catch (const std::exception& e) {
log::warning(logcat, "Error opening database: {}", e.what());
return 1;
}
r = core_storage->init(db, nullptr, nullptr, opt_testnet ? cryptonote::network_type::TESTNET : opt_devnet ? cryptonote::network_type::DEVNET : cryptonote::network_type::MAINNET);
r = core_storage->init(
db,
nullptr,
nullptr,
opt_testnet ? cryptonote::network_type::TESTNET
: opt_devnet ? cryptonote::network_type::DEVNET
: cryptonote::network_type::MAINNET);
if (core_storage->get_blockchain_pruning_seed() && !opt_blocks_dat)
{
if (core_storage->get_blockchain_pruning_seed() && !opt_blocks_dat) {
log::warning(logcat, "Blockchain is pruned, cannot export");
return 1;
}
@ -156,13 +158,10 @@ int main(int argc, char* argv[])
log::warning(logcat, "Source blockchain storage initialized OK");
log::warning(logcat, "Exporting blockchain raw data...");
if (opt_blocks_dat)
{
if (opt_blocks_dat) {
BlocksdatFile blocksdat;
r = blocksdat.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop);
}
else
{
} else {
BootstrapFile bootstrap;
r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop);
}

384
src/blockchain_utilities/blockchain_import.cpp
View file

@ -27,29 +27,29 @@
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <atomic>
#include <cstdio>
#include <fmt/color.h>
#include <unistd.h>
#include <algorithm>
#include <atomic>
#include <boost/algorithm/string.hpp>
#include <cstdio>
#include <fstream>
#include <boost/algorithm/string.hpp>
#include <unistd.h>
#include "blocks/blocks.h"
#include "bootstrap_file.h"
#include "bootstrap_serialization.h"
#include "common/fs-format.h"
#include "common/hex.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "cryptonote_core/cryptonote_core.h"
#include "cryptonote_core/uptime_proof.h"
#include "cryptonote_protocol/quorumnet.h"
#include "epee/misc_log_ex.h"
#include "logging/oxen_logger.h"
#include "bootstrap_file.h"
#include "bootstrap_serialization.h"
#include "blocks/blocks.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "serialization/binary_utils.h"
#include "cryptonote_core/uptime_proof.h"
#include "cryptonote_core/cryptonote_core.h"
#include "common/hex.h"
#include "common/fs-format.h"
#include <fmt/color.h>
namespace
{
namespace {
// CONFIG
bool opt_batch = true;
bool opt_verify = true; // use add_new_block, which does verification before calling add_block
@ -79,9 +79,7 @@ const command_line::arg_descriptor<bool> arg_recalculate_difficulty = {
// difficulty values (and the chain that kept going violated the correct difficulty, and got
// checkpointed multiple times because enough of the network followed it).
false};
}
} // namespace
namespace po = boost::program_options;
@ -90,8 +88,7 @@ using namespace cryptonote;
static auto logcat = log::Cat("bcutil");
// db_mode: safe, fast, fastest
int get_db_flags_from_mode(const std::string& db_mode)
{
int get_db_flags_from_mode(const std::string& db_mode) {
int db_flags = 0;
if (db_mode == "safe")
db_flags = DBF_SAFE;
@ -102,23 +99,19 @@ int get_db_flags_from_mode(const std::string& db_mode)
return db_flags;
}
int pop_blocks(cryptonote::core& core, int num_blocks)
{
int pop_blocks(cryptonote::core& core, int num_blocks) {
bool use_batch = opt_batch;
if (use_batch) core.get_blockchain_storage().get_db().batch_start();
try
{
core.get_blockchain_storage().pop_blocks(num_blocks);
if (use_batch)
{
core.get_blockchain_storage().get_db().batch_start();
try {
core.get_blockchain_storage().pop_blocks(num_blocks);
if (use_batch) {
core.get_blockchain_storage().get_db().batch_stop();
core.get_blockchain_storage().get_db().show_stats();
}
}
catch(const std::exception &e)
{
} catch (const std::exception& e) {
// There was an error, so don't commit pending data.
// Destructor will abort write txn.
}
@ -126,8 +119,7 @@ int pop_blocks(cryptonote::core& core, int num_blocks)
return num_blocks;
}
int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &blocks, bool force)
{
int check_flush(cryptonote::core& core, std::vector<block_complete_entry>& blocks, bool force) {
if (blocks.empty())
return 0;
if (!force && blocks.size() < db_batch_size)
@ -139,12 +131,13 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
return 0;
std::vector<crypto::hash> hashes;
for (const auto &b: blocks)
{
for (const auto& b : blocks) {
cryptonote::block block;
if (!parse_and_validate_block_from_blob(b.block, block))
{
log::error(logcat, "Failed to parse block: {}", tools::type_to_hex(get_blob_hash(b.block)));
if (!parse_and_validate_block_from_blob(b.block, block)) {
log::error(
logcat,
"Failed to parse block: {}",
tools::type_to_hex(get_blob_hash(b.block)));
core.cleanup_handle_incoming_blocks();
return 1;
}
@ -154,29 +147,27 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
// TODO(doyle): Checkpointing
std::vector<block> pblocks;
if (!core.prepare_handle_incoming_blocks(blocks, pblocks))
{
if (!core.prepare_handle_incoming_blocks(blocks, pblocks)) {
log::error(logcat, "Failed to prepare to add blocks");
return 1;
}
if (!pblocks.empty() && pblocks.size() != blocks.size())
{
if (!pblocks.empty() && pblocks.size() != blocks.size()) {
log::error(logcat, "Unexpected parsed blocks size");
core.cleanup_handle_incoming_blocks();
return 1;
}
size_t blockidx = 0;
for(const block_complete_entry& block_entry: blocks)
{
for (const block_complete_entry& block_entry : blocks) {
// process transactions
for(auto& tx_blob: block_entry.txs)
{
for (auto& tx_blob : block_entry.txs) {
tx_verification_context tvc{};
core.handle_incoming_tx(tx_blob, tvc, tx_pool_options::from_block());
if(tvc.m_verifivation_failed)
{
log::error(logcat, "transaction verification failed, tx_id = {}", tools::type_to_hex(get_blob_hash(tx_blob)));
if (tvc.m_verifivation_failed) {
log::error(
logcat,
"transaction verification failed, tx_id = {}",
tools::type_to_hex(get_blob_hash(tx_blob)));
core.cleanup_handle_incoming_blocks();
return 1;
}
@ -186,16 +177,22 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
block_verification_context bvc{};
core.handle_incoming_block(block_entry.block, pblocks.empty() ? NULL : &pblocks[blockidx++], bvc, nullptr /*checkpoint*/, false); // <--- process block
core.handle_incoming_block(
block_entry.block,
pblocks.empty() ? NULL : &pblocks[blockidx++],
bvc,
nullptr /*checkpoint*/,
false); // <--- process block
if(bvc.m_verifivation_failed)
{
log::error(logcat, "Block verification failed, id = {}", tools::type_to_hex(get_blob_hash(block_entry.block)));
if (bvc.m_verifivation_failed) {
log::error(
logcat,
"Block verification failed, id = {}",
tools::type_to_hex(get_blob_hash(block_entry.block)));
core.cleanup_handle_incoming_blocks();
return 1;
}
if(bvc.m_marked_as_orphaned)
{
if (bvc.m_marked_as_orphaned) {
log::error(logcat, "Block received at sync phase was marked as orphaned");
core.cleanup_handle_incoming_blocks();
return 1;
@ -209,15 +206,14 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
return 0;
}
int import_from_file(cryptonote::core& core, const fs::path& import_file_path, uint64_t block_stop=0)
{
int import_from_file(
cryptonote::core& core, const fs::path& import_file_path, uint64_t block_stop = 0) {
// Reset stats, in case we're using newly created db, accumulating stats
// from addition of genesis block.
// This aligns internal db counts with importer counts.
core.get_blockchain_storage().get_db().reset_stats();
if (std::error_code ec; !fs::exists(import_file_path, ec))
{
if (std::error_code ec; !fs::exists(import_file_path, ec)) {
log::error(logcat, "bootstrap file not found: {}", import_file_path);
return false;
}
@ -231,10 +227,13 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
std::streampos pos;
// BootstrapFile bootstrap(import_file_path);
uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path, pos, seek_height);
log::info(logcat, "bootstrap file last block number: {} (zero-based height) total blocks: {}", total_source_blocks-1, total_source_blocks);
log::info(
logcat,
"bootstrap file last block number: {} (zero-based height) total blocks: {}",
total_source_blocks - 1,
total_source_blocks);
if (total_source_blocks-1 <= start_height)
{
if (total_source_blocks - 1 <= start_height) {
return false;
}
@ -242,8 +241,7 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
fs::ifstream import_file{import_file_path, std::ios::binary};
if (import_file.fail())
{
if (import_file.fail()) {
log::error(logcat, "import_file.open() fail");
return false;
}
@ -261,8 +259,7 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
// Note that a new blockchain will start with block number 0 (total blocks: 1)
// due to genesis block being added at initialization.
if (! block_stop)
{
if (!block_stop) {
block_stop = total_source_blocks - 1;
}
@ -285,16 +282,14 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
bool q2 = false;
import_file.seekg(pos);
bytes_read = bootstrap.count_bytes(import_file, start_height - seek_height, h, q2);
if (q2)
{
if (q2) {
quit = 2;
goto quitting;
}
h = start_height;
}
if (use_batch)
{
if (use_batch) {
uint64_t bytes, h2;
bool q2;
pos = import_file.tellg();
@ -304,8 +299,7 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
import_file.seekg(pos);
core.get_blockchain_storage().get_db().batch_start(db_batch_size, bytes);
}
while (! quit)
{
while (!quit) {
uint32_t chunk_size;
import_file.read(buffer1, sizeof(chunk_size));
// TODO: bootstrap.read_chunk();
@ -324,49 +318,50 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
}
log::debug(logcat, "chunk_size: {}", chunk_size);
if (chunk_size > BUFFER_SIZE)
{
log::warning(logcat, "WARNING: chunk_size {} > BUFFER_SIZE {}", chunk_size, BUFFER_SIZE);
if (chunk_size > BUFFER_SIZE) {
log::warning(
logcat, "WARNING: chunk_size {} > BUFFER_SIZE {}", chunk_size, BUFFER_SIZE);
throw std::runtime_error("Aborting: chunk size exceeds buffer size");
}
if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD)
{
if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD) {
log::info(logcat, "NOTE: chunk_size {} > {}", chunk_size, CHUNK_SIZE_WARNING_THRESHOLD);
}
else if (chunk_size == 0) {
} else if (chunk_size == 0) {
log::error(logcat, "ERROR: chunk_size == 0");
return 2;
}
import_file.read(buffer_block, chunk_size);
if (!import_file) {
if (import_file.eof())
{
if (import_file.eof()) {
std::cout << refresh_string;
log::info(logcat, "End of file reached - file was truncated");
quit = 1;
break;
}
else
{
log::error(logcat, "ERROR: unexpected end of file: bytes read before error: {} of chunk_size {}", import_file.gcount(), chunk_size);
} else {
log::error(
logcat,
"ERROR: unexpected end of file: bytes read before error: {} of chunk_size "
"{}",
import_file.gcount(),
chunk_size);
return 2;
}
}
bytes_read += chunk_size;
log::debug(logcat, "Total bytes read: {}", bytes_read);
if (h > block_stop)
{
std::cout << refresh_string << "block " << h-1
<< " / " << block_stop
<< "\n" << std::endl;
log::info(logcat, "Specified block number reached - stopping. block: {} total blocks: {}", h-1, h);
if (h > block_stop) {
std::cout << refresh_string << "block " << h - 1 << " / " << block_stop << "\n"
<< std::endl;
log::info(
logcat,
"Specified block number reached - stopping. block: {} total blocks: {}",
h - 1,
h);
quit = 1;
break;
}
try
{
try {
bootstrap::block_package bp;
try {
serialization::parse_binary(std::string_view{buffer_block, chunk_size}, bp);
@ -376,49 +371,39 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
int display_interval = 1000;
int progress_interval = 10;
// NOTE: use of NUM_BLOCKS_PER_CHUNK is a placeholder in case multi-block chunks are later supported.
for (int chunk_ind = 0; chunk_ind < NUM_BLOCKS_PER_CHUNK; ++chunk_ind)
{
// NOTE: use of NUM_BLOCKS_PER_CHUNK is a placeholder in case multi-block chunks are
// later supported.
for (int chunk_ind = 0; chunk_ind < NUM_BLOCKS_PER_CHUNK; ++chunk_ind) {
++h;
if ((h-1) % display_interval == 0)
{
if ((h - 1) % display_interval == 0) {
std::cout << refresh_string;
log::debug(logcat, "loading block number {}", h - 1);
}
else
{
} else {
log::debug(logcat, "loading block number {}", h - 1);
}
b = bp.block;
log::debug(logcat, "block prev_id: {}\n", b.prev_id);
if ((h-1) % progress_interval == 0)
{
std::cout << refresh_string << "block " << h-1
<< " / " << block_stop
<< "\r" << std::flush;
if ((h - 1) % progress_interval == 0) {
std::cout << refresh_string << "block " << h - 1 << " / " << block_stop << "\r"
<< std::flush;
}
if (opt_verify)
{
if (opt_verify) {
std::string block;
cryptonote::block_to_blob(bp.block, block);
std::vector<std::string> txs;
for (const auto &tx: bp.txs)
{
for (const auto& tx : bp.txs) {
txs.push_back(std::string());
cryptonote::tx_to_blob(tx, txs.back());
}
blocks.push_back({block, txs});
int ret = check_flush(core, blocks, false);
if (ret)
{
if (ret) {
quit = 2; // make sure we don't commit partial block data
break;
}
}
else
{
} else {
std::vector<std::pair<transaction, std::string>> txs;
std::vector<transaction> archived_txs;
@ -426,8 +411,7 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
// tx number 1: coinbase tx
// tx number 2 onwards: archived_txs
for (const transaction &tx : archived_txs)
{
for (const transaction& tx : archived_txs) {
// add blocks with verification.
// for Blockchain and blockchain_storage add_new_block().
// for add_block() method, without (much) processing.
@ -447,23 +431,26 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
cumulative_difficulty = bp.cumulative_difficulty;
coins_generated = bp.coins_generated;
try
{
uint64_t long_term_block_weight = core.get_blockchain_storage().get_next_long_term_block_weight(block_weight);
core.get_blockchain_storage().get_db().add_block(std::make_pair(b, block_to_blob(b)), block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs);
}
catch (const std::exception& e)
{
try {
uint64_t long_term_block_weight =
core.get_blockchain_storage().get_next_long_term_block_weight(
block_weight);
core.get_blockchain_storage().get_db().add_block(
std::make_pair(b, block_to_blob(b)),
block_weight,
long_term_block_weight,
cumulative_difficulty,
coins_generated,
txs);
} catch (const std::exception& e) {
std::cout << refresh_string;
log::error(logcat, "Error adding block to blockchain: {}", e.what());
quit = 2; // make sure we don't commit partial block data
break;
}
if (use_batch)
{
if ((h-1) % db_batch_size == 0)
{
if (use_batch) {
if ((h - 1) % db_batch_size == 0) {
uint64_t bytes, h2;
bool q2;
std::cout << refresh_string;
@ -473,7 +460,8 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
pos = import_file.tellg();
bytes = bootstrap.count_bytes(import_file, db_batch_size, h2, q2);
import_file.seekg(pos);
core.get_blockchain_storage().get_db().batch_start(db_batch_size, bytes);
core.get_blockchain_storage().get_db().batch_start(
db_batch_size, bytes);
std::cout << "\n";
core.get_blockchain_storage().get_db().show_stats();
}
@ -481,9 +469,7 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
}
++num_imported;
}
}
catch (const std::exception& e)
{
} catch (const std::exception& e) {
std::cout << refresh_string;
log::error(logcat, "exception while reading from file, height={}: {}", h, e.what());
return 2;
@ -493,22 +479,17 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
quitting:
import_file.close();
if (opt_verify)
{
if (opt_verify) {
int ret = check_flush(core, blocks, true);
if (ret)
return ret;
}
if (use_batch)
{
if (quit > 1)
{
if (use_batch) {
if (quit > 1) {
// There was an error, so don't commit pending data.
// Destructor will abort write txn.
}
else
{
} else {
core.get_blockchain_storage().get_db().batch_stop();
}
}
@ -523,8 +504,7 @@ quitting:
return 0;
}
int main(int argc, char* argv[])
{
int main(int argc, char* argv[]) {
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
@ -539,23 +519,28 @@ int main(int argc, char* argv[])
auto opt_size = command_line::boost_option_sizes();
po::options_description desc_cmd_only("Command line options", opt_size.first, opt_size.second);
po::options_description desc_cmd_sett("Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_input_file = {"input-file", "Specify input file", "", true};
const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop};
po::options_description desc_cmd_sett(
"Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_input_file = {
"input-file", "Specify input file", "", true};
const command_line::arg_descriptor<std::string> arg_log_level = {
"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<uint64_t> arg_block_stop = {
"block-stop", "Stop at block number", block_stop};
const command_line::arg_descriptor<uint64_t> arg_batch_size = {"batch-size", "", db_batch_size};
const command_line::arg_descriptor<uint64_t> arg_pop_blocks = {"pop-blocks", "Remove blocks from end of blockchain", num_blocks};
const command_line::arg_descriptor<uint64_t> arg_pop_blocks = {
"pop-blocks", "Remove blocks from end of blockchain", num_blocks};
const command_line::arg_descriptor<bool> arg_count_blocks = {
"count-blocks"
, "Count blocks in bootstrap file and exit"
, false
};
const command_line::arg_descriptor<bool> arg_noverify = {"dangerous-unverified-import",
"Blindly trust the import file and use potentially malicious blocks and transactions during import (only enable if you exported the file yourself)", false};
const command_line::arg_descriptor<bool> arg_batch = {"batch",
"Batch transactions for faster import", true};
const command_line::arg_descriptor<bool> arg_resume = {"resume",
"Resume from current height if output database already exists", true};
"count-blocks", "Count blocks in bootstrap file and exit", false};
const command_line::arg_descriptor<bool> arg_noverify = {
"dangerous-unverified-import",
"Blindly trust the import file and use potentially malicious blocks and transactions "
"during import (only enable if you exported the file yourself)",
false};
const command_line::arg_descriptor<bool> arg_batch = {
"batch", "Batch transactions for faster import", true};
const command_line::arg_descriptor<bool> arg_resume = {
"resume", "Resume from current height if output database already exists", true};
command_line::add_arg(desc_cmd_sett, arg_input_file);
command_line::add_arg(desc_cmd_sett, arg_log_level);
@ -570,19 +555,17 @@ int main(int argc, char* argv[])
// call add_options() directly for these arguments since
// command_line helpers support only boolean switch, not boolean argument
desc_cmd_sett.add_options()
(arg_noverify.name, make_semantic(arg_noverify), arg_noverify.description)
(arg_batch.name, make_semantic(arg_batch), arg_batch.description)
(arg_resume.name, make_semantic(arg_resume), arg_resume.description)
;
desc_cmd_sett.add_options()(
arg_noverify.name, make_semantic(arg_noverify), arg_noverify.description)(
arg_batch.name, make_semantic(arg_batch), arg_batch.description)(
arg_resume.name, make_semantic(arg_resume), arg_resume.description);
po::options_description desc_options("Allowed options");
desc_options.add(desc_cmd_only).add(desc_cmd_sett);
cryptonote::core::init_options(desc_options);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
{
bool r = command_line::handle_error_helper(desc_options, [&]() {
po::store(po::parse_command_line(argc, argv, desc_options), vm);
po::notify(vm);
return true;
@ -596,40 +579,34 @@ int main(int argc, char* argv[])
block_stop = command_line::get_arg(vm, arg_block_stop);
db_batch_size = command_line::get_arg(vm, arg_batch_size);
if (command_line::get_arg(vm, command_line::arg_help))
{
if (command_line::get_arg(vm, command_line::arg_help)) {
std::cout << "Oxen '" << OXEN_RELEASE_NAME << "' (v" << OXEN_VERSION_FULL << ")\n\n";
std::cout << desc_options << std::endl;
return 1;
}
if (! opt_batch && !command_line::is_arg_defaulted(vm, arg_batch_size))
{
if (!opt_batch && !command_line::is_arg_defaulted(vm, arg_batch_size)) {
std::cerr << "Error: batch-size set, but batch option not enabled\n";
return 1;
}
if (! db_batch_size)
{
if (!db_batch_size) {
std::cerr << "Error: batch-size must be > 0\n";
return 1;
}
if (opt_verify && command_line::is_arg_defaulted(vm, arg_batch_size))
{
if (opt_verify && command_line::is_arg_defaulted(vm, arg_batch_size)) {
// usually want batch size default lower if verify on, so progress can be
// frequently saved.
//
// currently, with Windows, default batch size is low, so ignore
// default db_batch_size_verify unless it's even lower
if (db_batch_size > db_batch_size_verify)
{
if (db_batch_size > db_batch_size_verify) {
db_batch_size = db_batch_size_verify;
}
}
opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
if (opt_testnet && opt_devnet)
{
if (opt_testnet && opt_devnet) {
std::cerr << "Error: Can't specify more than one of --testnet and --devnet\n";
return 1;
}
@ -639,7 +616,8 @@ int main(int argc, char* argv[])
if (auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str()
<< std::endl;
throw std::runtime_error{"Incorrect log level"};
}
@ -653,8 +631,7 @@ int main(int argc, char* argv[])
else
import_file_path = fs::u8path(m_config_folder) / "export" / BLOCKCHAIN_RAW;
if (command_line::has_arg(vm, arg_count_blocks))
{
if (command_line::has_arg(vm, arg_count_blocks)) {
BootstrapFile bootstrap;
bootstrap.count_blocks(import_file_path);
return 0;
@ -662,12 +639,9 @@ int main(int argc, char* argv[])
log::info(logcat, "database: LMDB");
log::info(logcat, "verify: {}", opt_verify);
if (opt_batch)
{
if (opt_batch) {
log::info(logcat, "batch: {} batch size: {}", opt_batch, db_batch_size);
}
else
{
} else {
log::info(logcat, "batch: {}", opt_batch);
}
log::info(logcat, "resume: {}", opt_resume);
@ -676,23 +650,26 @@ int main(int argc, char* argv[])
log::info(logcat, "bootstrap file path: {}", import_file_path);
log::info(logcat, "database path: {}", m_config_folder);
if (!opt_verify)
{
log::warning(logcat, fg(fmt::terminal_color::red), "\n\
if (!opt_verify) {
log::warning(
logcat,
fg(fmt::terminal_color::red),
"\n\
Import is set to proceed WITHOUT VERIFICATION.\n\
This is a DANGEROUS operation: if the file was tampered with in transit, or obtained from a malicious source,\n\
you could end up with a compromised database. It is recommended to NOT use {}.\n\
*****************************************************************************************\n\
You have 90 seconds to press ^C or terminate this program before unverified import starts\n\
*****************************************************************************************", arg_noverify.name);
*****************************************************************************************",
arg_noverify.name);
sleep(90);
}
//TODO: currently using cryptonote_protocol stub for this kind of test, use real validation of relayed objects
// TODO: currently using cryptonote_protocol stub for this kind of test, use real validation of
// relayed objects
cryptonote::core core;
try
{
try {
#if defined(PER_BLOCK_CHECKPOINT)
const GetCheckpointsCallback& get_checkpoints = blocks::GetCheckpointsData;
@ -702,19 +679,23 @@ int main(int argc, char* argv[])
quorumnet::init_core_callbacks();
if (!core.init(vm, nullptr, get_checkpoints))
{
if (!core.init(vm, nullptr, get_checkpoints)) {
std::cerr << "Failed to initialize core\n";
return 1;
}
core.get_blockchain_storage().get_db().set_batch_transactions(true);
if (!command_line::is_arg_defaulted(vm, arg_pop_blocks))
{
if (!command_line::is_arg_defaulted(vm, arg_pop_blocks)) {
num_blocks = command_line::get_arg(vm, arg_pop_blocks);
log::info(logcat, "height: {}", core.get_blockchain_storage().get_current_blockchain_height());
log::info(
logcat,
"height: {}",
core.get_blockchain_storage().get_current_blockchain_height());
pop_blocks(core, num_blocks);
log::info(logcat, "height: {}", core.get_blockchain_storage().get_current_blockchain_height());
log::info(
logcat,
"height: {}",
core.get_blockchain_storage().get_current_blockchain_height());
return 0;
}
@ -728,10 +709,9 @@ int main(int argc, char* argv[])
// - disk sync if needed
//
core.deinit();
}
catch (const DB_ERROR& e)
{
std::cout << std::string("Error loading blockchain db: ") + e.what() + " -- shutting down now\n";
} catch (const DB_ERROR& e) {
std::cout << std::string("Error loading blockchain db: ") + e.what() +
" -- shutting down now\n";
core.deinit();
return 1;
}

5
src/blockchain_utilities/blockchain_objects.h
View file

@ -2,13 +2,12 @@
#define BLOCKCHAIN_OBJECTS_H
#include "cryptonote_core/blockchain.h"
#include "cryptonote_core/tx_pool.h"
#include "cryptonote_core/service_node_list.h"
#include "cryptonote_core/service_node_voting.h"
#include "cryptonote_core/tx_pool.h"
// NOTE(oxen): This is done this way because of the circular constructors.
struct blockchain_objects_t
{
struct blockchain_objects_t {
cryptonote::Blockchain m_blockchain;
cryptonote::tx_memory_pool m_mempool;
service_nodes::service_node_list m_service_node_list;

488
src/blockchain_utilities/blockchain_prune.cpp
View file

@ -26,17 +26,19 @@
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <array>
#include <lmdb.h>
#include "common/command_line.h"
#include "common/pruning.h"
#include "common/string_util.h"
#include "common/fs.h"
#include "cryptonote_core/cryptonote_core.h"
#include "cryptonote_core/blockchain.h"
#include <array>
#include "blockchain_db/blockchain_db.h"
#include "blockchain_db/lmdb/db_lmdb.h"
#include "blockchain_objects.h"
#include "common/command_line.h"
#include "common/fs.h"
#include "common/pruning.h"
#include "common/string_util.h"
#include "cryptonote_core/blockchain.h"
#include "cryptonote_core/cryptonote_core.h"
#include "version.h"
#define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void*)&val}
@ -52,16 +54,20 @@ static fs::path db_path;
static uint64_t records_per_sync = 128;
static const size_t slack = 512 * 1024 * 1024;
static std::error_code replace_file(const fs::path& replacement_name, const fs::path& replaced_name)
{
static std::error_code replace_file(
const fs::path& replacement_name, const fs::path& replaced_name) {
std::error_code ec = fs::rename(replacement_name, replaced_name);
if (ec)
log::error(logcat, "Error renaming {} to {}: {}", replacement_name, replaced_name, ec.message());
log::error(
logcat,
"Error renaming {} to {}: {}",
replacement_name,
replaced_name,
ec.message());
return ec;
}
static void open(MDB_env *&env, const fs::path &path, uint64_t db_flags, bool readonly)
{
static void open(MDB_env*& env, const fs::path& path, uint64_t db_flags, bool readonly) {
int dbr;
int flags = 0;
@ -73,33 +79,34 @@ static void open(MDB_env *&env, const fs::path &path, uint64_t db_flags, bool re
flags |= MDB_RDONLY;
dbr = mdb_env_create(&env);
if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
dbr = mdb_env_set_maxdbs(env, 32);
if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
dbr = mdb_env_open(env, path.string().c_str(), flags, 0664);
if (dbr) throw std::runtime_error("Failed to open database file '"
+ path.string() + "': " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to open database file '" + path.string() +
"': " + std::string(mdb_strerror(dbr)));
}
static void close(MDB_env *env)
{
static void close(MDB_env* env) {
mdb_env_close(env);
}
static void add_size(MDB_env *env, uint64_t bytes)
{
try
{
static void add_size(MDB_env* env, uint64_t bytes) {
try {
auto si = fs::space(db_path);
if(si.available < bytes)
{
log::error(logcat, "!! WARNING: Insufficient free space to extend database !!: ",
if (si.available < bytes) {
log::error(
logcat,
"!! WARNING: Insufficient free space to extend database !!: ",
(si.available >> 20L) << " MB available, " << (bytes >> 20L) << " MB needed");
return;
}
}
catch(...)
{
} catch (...) {
// print something but proceed.
log::warning(logcat, "Unable to query free disk space.");
}
@ -114,13 +121,18 @@ static void add_size(MDB_env *env, uint64_t bytes)
int result = mdb_env_set_mapsize(env, new_mapsize);
if (result)
throw std::runtime_error("Failed to set new mapsize to " + std::to_string(new_mapsize) + ": " + std::string(mdb_strerror(result)));
throw std::runtime_error(
"Failed to set new mapsize to " + std::to_string(new_mapsize) + ": " +
std::string(mdb_strerror(result)));
log::info(logcat, "LMDB Mapsize increased. Old: {}MiB, New: {}MiB", mei.me_mapsize / (1024 * 1024), new_mapsize / (1024 * 1024));
log::info(
logcat,
"LMDB Mapsize increased. Old: {}MiB, New: {}MiB",
mei.me_mapsize / (1024 * 1024),
new_mapsize / (1024 * 1024));
}
static void check_resize(MDB_env *env, size_t bytes)
{
static void check_resize(MDB_env* env, size_t bytes) {
MDB_envinfo mei;
MDB_stat mst;
@ -132,21 +144,28 @@ static void check_resize(MDB_env *env, size_t bytes)
add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize);
}
static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes)
{
static bool resize_point(size_t nrecords, MDB_env* env, MDB_txn** txn, size_t& bytes) {
if (nrecords % records_per_sync && bytes <= slack / 2)
return false;
int dbr = mdb_txn_commit(*txn);
if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr)));
check_resize(env, bytes);
dbr = mdb_txn_begin(env, NULL, 0, txn);
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
bytes = 0;
return true;
}
static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0)
{
static void copy_table(
MDB_env* env0,
MDB_env* env1,
const char* table,
unsigned int flags,
unsigned int putflags,
int (*cmp)(const MDB_val*, const MDB_val*) = 0) {
MDB_dbi dbi0, dbi1;
MDB_txn *txn0, *txn1;
MDB_cursor *cur0, *cur1;
@ -156,69 +175,95 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned
log::info(logcat, "Copying {}", table);
OXEN_DEFER {
if (tx_active1) mdb_txn_abort(txn1);
if (tx_active0) mdb_txn_abort(txn0);
if (tx_active1)
mdb_txn_abort(txn1);
if (tx_active0)
mdb_txn_abort(txn0);
};
dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0);
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
tx_active0 = true;
dbr = mdb_txn_begin(env1, NULL, 0, &txn1);
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
tx_active1 = true;
dbr = mdb_dbi_open(txn0, table, flags, &dbi0);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
if (cmp)
((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn0, dbi0, cmp);
dbr = mdb_dbi_open(txn1, table, flags, &dbi1);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
if (cmp)
((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn1, dbi1, cmp);
dbr = mdb_txn_commit(txn1);
if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr)));
tx_active1 = false;
MDB_stat stats;
dbr = mdb_env_stat(env0, &stats);
if (dbr) throw std::runtime_error("Failed to stat " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr)));
check_resize(env1, (stats.ms_branch_pages + stats.ms_overflow_pages + stats.ms_leaf_pages) * stats.ms_psize);
if (dbr)
throw std::runtime_error(
"Failed to stat " + std::string(table) +
" LMDB table: " + std::string(mdb_strerror(dbr)));
check_resize(
env1,
(stats.ms_branch_pages + stats.ms_overflow_pages + stats.ms_leaf_pages) *
stats.ms_psize);
dbr = mdb_txn_begin(env1, NULL, 0, &txn1);
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
tx_active1 = true;
dbr = mdb_drop(txn1, dbi1, 0);
if (dbr) throw std::runtime_error("Failed to empty " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to empty " + std::string(table) +
" LMDB table: " + std::string(mdb_strerror(dbr)));
dbr = mdb_cursor_open(txn0, dbi0, &cur0);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_cursor_open(txn1, dbi1, &cur1);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
MDB_val k;
MDB_val v;
MDB_cursor_op op = MDB_FIRST;
size_t nrecords = 0, bytes = 0;
while (1)
{
while (1) {
int ret = mdb_cursor_get(cur0, &k, &v, op);
op = MDB_NEXT;
if (ret == MDB_NOTFOUND)
break;
if (ret)
throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret)));
throw std::runtime_error(
"Failed to enumerate " + std::string(table) +
" records: " + std::string(mdb_strerror(ret)));
bytes += k.mv_size + v.mv_size;
if (resize_point(++nrecords, env1, &txn1, bytes))
{
if (resize_point(++nrecords, env1, &txn1, bytes)) {
dbr = mdb_cursor_open(txn1, dbi1, &cur1);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
}
ret = mdb_cursor_put(cur1, &k, &v, putflags);
if (ret)
throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret)));
throw std::runtime_error(
"Failed to write " + std::string(table) +
" record: " + std::string(mdb_strerror(ret)));
}
mdb_cursor_close(cur1);
@ -231,134 +276,171 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned
mdb_dbi_close(env0, dbi0);
}
static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id)
{
static bool is_v1_tx(MDB_cursor* c_txs_pruned, MDB_val* tx_id) {
MDB_val v;
int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET);
if (ret)
throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret)));
throw std::runtime_error(
"Failed to find transaction pruned data: " + std::string(mdb_strerror(ret)));
if (v.mv_size == 0)
throw std::runtime_error("Invalid transaction pruned data");
return cryptonote::is_v1_tx(std::string_view{(const char*)v.mv_data, v.mv_size});
}
static void prune(MDB_env *env0, MDB_env *env1)
{
MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties;
static void prune(MDB_env* env0, MDB_env* env1) {
MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable,
dbi1_txs_prunable_tip, dbi1_properties;
MDB_txn *txn0, *txn1;
MDB_cursor *cur0_txs_pruned, *cur0_txs_prunable, *cur0_tx_indices, *cur1_txs_prunable, *cur1_txs_prunable_tip;
MDB_cursor *cur0_txs_pruned, *cur0_txs_prunable, *cur0_tx_indices, *cur1_txs_prunable,
*cur1_txs_prunable_tip;
bool tx_active0 = false, tx_active1 = false;
int dbr;
log::info(logcat, "Creating pruned txs_prunable");
OXEN_DEFER {
if (tx_active1) mdb_txn_abort(txn1);
if (tx_active0) mdb_txn_abort(txn0);
if (tx_active1)
mdb_txn_abort(txn1);
if (tx_active0)
mdb_txn_abort(txn0);
};
dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0);
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
tx_active0 = true;
dbr = mdb_txn_begin(env1, NULL, 0, &txn1);
if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
tx_active1 = true;
dbr = mdb_dbi_open(txn0, "txs_pruned", MDB_INTEGERKEY, &dbi0_txs_pruned);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
mdb_set_compare(txn0, dbi0_txs_pruned, BlockchainLMDB::compare_uint64);
dbr = mdb_cursor_open(txn0, dbi0_txs_pruned, &cur0_txs_pruned);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(txn0, "txs_prunable", MDB_INTEGERKEY, &dbi0_txs_prunable);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
mdb_set_compare(txn0, dbi0_txs_prunable, BlockchainLMDB::compare_uint64);
dbr = mdb_cursor_open(txn0, dbi0_txs_prunable, &cur0_txs_prunable);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(txn0, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi0_tx_indices);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(
txn0, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi0_tx_indices);
if (dbr)
throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
mdb_set_dupsort(txn0, dbi0_tx_indices, BlockchainLMDB::compare_hash32);
dbr = mdb_cursor_open(txn0, dbi0_tx_indices, &cur0_tx_indices);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(txn1, "txs_prunable", MDB_INTEGERKEY, &dbi1_txs_prunable);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
mdb_set_compare(txn1, dbi1_txs_prunable, BlockchainLMDB::compare_uint64);
dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(txn1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi1_txs_prunable_tip);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(
txn1,
"txs_prunable_tip",
MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED,
&dbi1_txs_prunable_tip);
if (dbr)
throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
mdb_set_dupsort(txn1, dbi1_txs_prunable_tip, BlockchainLMDB::compare_uint64);
dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_drop(txn1, dbi1_txs_prunable, 0);
if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr)));
dbr = mdb_drop(txn1, dbi1_txs_prunable_tip, 0);
if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr)));
dbr = mdb_dbi_open(txn1, "properties", 0, &dbi1_properties);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
MDB_val k, v;
uint32_t pruning_seed = tools::make_pruning_seed(tools::get_random_stripe(), PRUNING_LOG_STRIPES);
uint32_t pruning_seed =
tools::make_pruning_seed(tools::get_random_stripe(), PRUNING_LOG_STRIPES);
static char pruning_seed_key[] = "pruning_seed";
k.mv_data = pruning_seed_key;
k.mv_size = strlen("pruning_seed") + 1;
v.mv_data = (void*)&pruning_seed;
v.mv_size = sizeof(pruning_seed);
dbr = mdb_put(txn1, dbi1_properties, &k, &v, 0);
if (dbr) throw std::runtime_error("Failed to save pruning seed: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to save pruning seed: " + std::string(mdb_strerror(dbr)));
MDB_stat stats;
dbr = mdb_dbi_open(txn0, "blocks", 0, &dbi0_blocks);
if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
dbr = mdb_stat(txn0, dbi0_blocks, &stats);
if (dbr) throw std::runtime_error("Failed to query size of blocks: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to query size of blocks: " + std::string(mdb_strerror(dbr)));
mdb_dbi_close(env0, dbi0_blocks);
const uint64_t blockchain_height = stats.ms_entries;
size_t nrecords = 0, bytes = 0;
MDB_cursor_op op = MDB_FIRST;
while (1)
{
while (1) {
int ret = mdb_cursor_get(cur0_tx_indices, &k, &v, op);
op = MDB_NEXT;
if (ret == MDB_NOTFOUND)
break;
if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret)));
if (ret)
throw std::runtime_error(
"Failed to enumerate records: " + std::string(mdb_strerror(ret)));
const txindex* ti = (const txindex*)v.mv_data;
const uint64_t block_height = ti->data.block_id;
MDB_val_set(kk, ti->data.tx_id);
if (block_height + PRUNING_TIP_BLOCKS >= blockchain_height)
{
if (block_height + PRUNING_TIP_BLOCKS >= blockchain_height) {
log::debug(logcat, "{}/{} is in tip", block_height, blockchain_height);
MDB_val_set(vv, block_height);
dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0);
if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr)));
bytes += kk.mv_size + vv.mv_size;
}
if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk))
{
if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) ||
is_v1_tx(cur0_txs_pruned, &kk)) {
MDB_val vv;
dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET);
if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to read prunable tx data: " + std::string(mdb_strerror(dbr)));
bytes += kk.mv_size + vv.mv_size;
if (resize_point(++nrecords, env1, &txn1, bytes))
{
if (resize_point(++nrecords, env1, &txn1, bytes)) {
dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip);
if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
if (dbr)
throw std::runtime_error(
"Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr)));
}
dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0);
if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr)));
}
else
{
if (dbr)
throw std::runtime_error(
"Failed to write prunable tx data: " + std::string(mdb_strerror(dbr)));
} else {
log::debug(logcat, "{}/{} should be pruned, dropping", block_height, blockchain_height);
}
}
@ -380,8 +462,7 @@ static void prune(MDB_env *env0, MDB_env *env1)
mdb_dbi_close(env0, dbi0_tx_indices);
}
static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags)
{
static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t& db_flags) {
auto options = tools::split_any(db_sync_mode, " :", true);
for (const auto& option : options)
@ -392,35 +473,26 @@ static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags)
db_flags = 0;
if(options.size() == 0)
{
if (options.size() == 0) {
// default to fast:async:1
db_flags = DEFAULT_FLAGS;
}
bool safemode = false;
if(options.size() >= 1)
{
if(options[0] == "safe")
{
if (options.size() >= 1) {
if (options[0] == "safe") {
safemode = true;
db_flags = DBF_SAFE;
}
else if(options[0] == "fast")
{
} else if (options[0] == "fast") {
db_flags = DBF_FAST;
}
else if(options[0] == "fastest")
{
} else if (options[0] == "fastest") {
db_flags = DBF_FASTEST;
records_per_sync = 1000; // default to fastest:async:1000
}
else
} else
return false;
}
if(options.size() >= 2 && !safemode)
{
if (options.size() >= 2 && !safemode) {
char* endptr;
std::string bpsstr{options[1]};
uint64_t bps = strtoull(bpsstr.c_str(), &endptr, 0);
@ -432,8 +504,7 @@ static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags)
return true;
}
int main(int argc, char* argv[])
{
int main(int argc, char* argv[]) {
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
@ -445,14 +516,16 @@ int main(int argc, char* argv[])
auto opt_size = command_line::boost_option_sizes();
po::options_description desc_cmd_only("Command line options", opt_size.first, opt_size.second);
po::options_description desc_cmd_sett("Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
po::options_description desc_cmd_sett(
"Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_log_level = {
"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<std::string> arg_db_sync_mode = {
"db-sync-mode"
, "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]."
, "fast:1000"
};
const command_line::arg_descriptor<bool> arg_copy_pruned_database = {"copy-pruned-database", "Copy database anyway if already pruned"};
"db-sync-mode",
"Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync].",
"fast:1000"};
const command_line::arg_descriptor<bool> arg_copy_pruned_database = {
"copy-pruned-database", "Copy database anyway if already pruned"};
command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
@ -466,8 +539,7 @@ int main(int argc, char* argv[])
desc_options.add(desc_cmd_only).add(desc_cmd_sett);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
{
bool r = command_line::handle_error_helper(desc_options, [&]() {
auto parser = po::command_line_parser(argc, argv).options(desc_options);
po::store(parser.run(), vm);
po::notify(vm);
@ -476,8 +548,7 @@ int main(int argc, char* argv[])
if (!r)
return 1;
if (command_line::get_arg(vm, command_line::arg_help))
{
if (command_line::get_arg(vm, command_line::arg_help)) {
std::cout << "Oxen '" << OXEN_RELEASE_NAME << "' (v" << OXEN_VERSION_FULL << ")\n\n";
std::cout << desc_options << std::endl;
return 1;
@ -493,7 +564,9 @@ int main(int argc, char* argv[])
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
network_type net_type = opt_testnet ? network_type::TESTNET : opt_devnet ? network_type::DEVNET : network_type::MAINNET;
network_type net_type = opt_testnet ? network_type::TESTNET
: opt_devnet ? network_type::DEVNET
: network_type::MAINNET;
bool opt_copy_pruned_database = command_line::get_arg(vm, arg_copy_pruned_database);
std::string data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir);
while (tools::ends_with(data_dir, "/") || tools::ends_with(data_dir, "\\"))
@ -501,8 +574,7 @@ int main(int argc, char* argv[])
std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode);
uint64_t db_flags = 0;
if (!parse_db_sync_mode(db_sync_mode, db_flags))
{
if (!parse_db_sync_mode(db_sync_mode, db_flags)) {
log::error(logcat, "Invalid db sync mode: {}", db_sync_mode);
return 1;
}
@ -511,7 +583,8 @@ int main(int argc, char* argv[])
// Use Blockchain instead of lower-level BlockchainDB for two reasons:
// 1. Blockchain has the init() method for easy setup
// 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash()
// 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(),
// get_block_by_hash()
//
// cannot match blockchain_storage setup above with just one line,
// e.g.
@ -522,52 +595,42 @@ int main(int argc, char* argv[])
std::array<Blockchain*, 2> core_storage;
fs::path paths[2];
bool already_pruned = false;
for (size_t n = 0; n < core_storage.size(); ++n)
{
for (size_t n = 0; n < core_storage.size(); ++n) {
blockchain_objects_t* blockchain_objects = new blockchain_objects_t();
core_storage[n] = &(blockchain_objects->m_blockchain);
BlockchainDB* db = new_db();
if (db == NULL)
{
if (db == NULL) {
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
if (n == 1)
{
if (n == 1) {
paths[1] = fs::u8path(data_dir) / (db->get_db_name() + "-pruned");
if (fs::exists(paths[1]))
{
if (!fs::is_directory(paths[1]))
{
log::error(logcat, "LMDB needs a directory path, but a file was passed: {}", paths[1].string());
if (fs::exists(paths[1])) {
if (!fs::is_directory(paths[1])) {
log::error(
logcat,
"LMDB needs a directory path, but a file was passed: {}",
paths[1].string());
return 1;
}
}
else
{
if (!fs::create_directories(paths[1]))
{
} else {
if (!fs::create_directories(paths[1])) {
log::error(logcat, "Failed to create directory: {}", paths[1].string());
return 1;
}
}
db_path = paths[1];
}
else
{
} else {
paths[0] = fs::u8path(data_dir) / db->get_db_name();
}
log::info(logcat, "Loading blockchain from folder {} ...", paths[n]);
try
{
try {
db->open(paths[n].string(), core_storage[n]->nettype(), n == 0 ? DBF_RDONLY : 0);
}
catch (const std::exception& e)
{
} catch (const std::exception& e) {
log::error(logcat, "Error opening database: {}", e.what());
return 1;
}
@ -576,11 +639,12 @@ int main(int argc, char* argv[])
std::string source_dest = n == 0 ? "source" : "pruned";
CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize " << source_dest << " blockchain storage");
log::info(logcat, "{} blockchain storage initialized OK", source_dest);
if (n == 0 && core_storage[0]->get_blockchain_pruning_seed())
{
if (!opt_copy_pruned_database)
{
log::error(logcat, "Blockchain is already pruned, use --{} to copy it anyway", arg_copy_pruned_database.name);
if (n == 0 && core_storage[0]->get_blockchain_pruning_seed()) {
if (!opt_copy_pruned_database) {
log::error(
logcat,
"Blockchain is already pruned, use --{} to copy it anyway",
arg_copy_pruned_database.name);
return 1;
}
already_pruned = true;
@ -596,38 +660,90 @@ int main(int argc, char* argv[])
open(env0, paths[0], db_flags, true);
open(env1, paths[1], db_flags, false);
copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND);
copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
copy_table(
env0,
env1,
"block_info",
MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED,
MDB_APPENDDUP,
BlockchainLMDB::compare_uint64);
copy_table(
env0,
env1,
"block_heights",
MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED,
0,
BlockchainLMDB::compare_hash32);
// copy_table(env0, env1, "txs", MDB_INTEGERKEY);
copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND);
copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND);
copy_table(
env0,
env1,
"txs_prunable_hash",
MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED,
MDB_APPEND);
// not copied: prunable, prunable_tip
copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32);
copy_table(
env0,
env1,
"tx_indices",
MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED,
0,
BlockchainLMDB::compare_hash32);
copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND);
copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64);
copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
copy_table(
env0,
env1,
"output_txs",
MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED,
MDB_APPENDDUP,
BlockchainLMDB::compare_uint64);
copy_table(
env0,
env1,
"output_amounts",
MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED,
MDB_APPENDDUP,
BlockchainLMDB::compare_uint64);
copy_table(
env0,
env1,
"spent_keys",
MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED,
MDB_NODUPDATA,
BlockchainLMDB::compare_hash32);
copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32);
copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND);
copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string);
if (already_pruned)
{
copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64);
copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64);
}
else
{
if (already_pruned) {
copy_table(
env0,
env1,
"txs_prunable",
MDB_INTEGERKEY,
MDB_APPEND,
BlockchainLMDB::compare_uint64);
copy_table(
env0,
env1,
"txs_prunable_tip",
MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED,
MDB_NODUPDATA,
BlockchainLMDB::compare_uint64);
} else {
prune(env0, env1);
}
close(env1);
close(env0);
log::info(logcat, "Swapping databases, pre-pruning blockchain will be left in {}", paths[0].string() + "-old and can be removed if desired");
log::info(
logcat,
"Swapping databases, pre-pruning blockchain will be left in {}",
paths[0].string() + "-old and can be removed if desired");
fs::path old = paths[0];
old += "-old";
if (replace_file(paths[0], old) || replace_file(paths[1], paths[0]))
{
if (replace_file(paths[0], old) || replace_file(paths[1], paths[0])) {
log::error(logcat, "Blockchain pruned OK, but renaming failed");
return 1;
}

122
src/blockchain_utilities/blockchain_prune_known_spent_data.cpp
View file

@ -30,11 +30,11 @@
#define __STDC_FORMAT_MACROS // NOTE(oxen): Explicitly define the PRIu64 macro on Mingw
#endif
#include "common/command_line.h"
#include "serialization/crypto.h"
#include "cryptonote_core/cryptonote_core.h"
#include "blockchain_objects.h"
#include "blockchain_db/blockchain_db.h"
#include "blockchain_objects.h"
#include "common/command_line.h"
#include "cryptonote_core/cryptonote_core.h"
#include "serialization/crypto.h"
#include "version.h"
namespace po = boost::program_options;
@ -42,8 +42,7 @@ using namespace cryptonote;
static auto logcat = log::Cat("bcutil");
static std::map<uint64_t, uint64_t> load_outputs(const fs::path& filename)
{
static std::map<uint64_t, uint64_t> load_outputs(const fs::path& filename) {
std::map<uint64_t, uint64_t> outputs;
uint64_t amount = std::numeric_limits<uint64_t>::max();
@ -54,13 +53,11 @@ static std::map<uint64_t, uint64_t> load_outputs(const fs::path& filename)
fopen(filename.c_str(), "r");
#endif
if (!f)
{
if (!f) {
log::error(logcat, "Failed to load outputs from {}: {}", filename, strerror(errno));
return {};
}
while (1)
{
while (1) {
char s[256];
if (!fgets(s, sizeof(s), f))
break;
@ -73,25 +70,19 @@ static std::map<uint64_t, uint64_t> load_outputs(const fs::path& filename)
continue;
std::pair<uint64_t, uint64_t> output;
uint64_t offset, num_offsets;
if (sscanf(s, "@%" PRIu64, &amount) == 1)
{
if (sscanf(s, "@%" PRIu64, &amount) == 1) {
continue;
}
if (amount == std::numeric_limits<uint64_t>::max())
{
if (amount == std::numeric_limits<uint64_t>::max()) {
log::error(logcat, "Bad format in {}", filename);
continue;
}
if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits<uint64_t>::max() - offset)
{
if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 &&
num_offsets < std::numeric_limits<uint64_t>::max() - offset) {
outputs[amount] += num_offsets;
}
else if (sscanf(s, "%" PRIu64, &offset) == 1)
{
} else if (sscanf(s, "%" PRIu64, &offset) == 1) {
outputs[amount] += 1;
}
else
{
} else {
log::error(logcat, "Bad format in {}", filename);
continue;
}
@ -100,8 +91,7 @@ static std::map<uint64_t, uint64_t> load_outputs(const fs::path& filename)
return outputs;
}
int main(int argc, char* argv[])
{
int main(int argc, char* argv[]) {
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
@ -113,11 +103,15 @@ int main(int argc, char* argv[])
auto opt_size = command_line::boost_option_sizes();
po::options_description desc_cmd_only("Command line options", opt_size.first, opt_size.second);
po::options_description desc_cmd_sett("Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
po::options_description desc_cmd_sett(
"Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_log_level = {
"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<bool> arg_verbose = {"verbose", "Verbose output", false};
const command_line::arg_descriptor<bool> arg_dry_run = {"dry-run", "Do not actually prune", false};
const command_line::arg_descriptor<std::string> arg_input = {"input", "Path to the known spent outputs file"};
const command_line::arg_descriptor<bool> arg_dry_run = {
"dry-run", "Do not actually prune", false};
const command_line::arg_descriptor<std::string> arg_input = {
"input", "Path to the known spent outputs file"};
command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
@ -132,8 +126,7 @@ int main(int argc, char* argv[])
desc_options.add(desc_cmd_only).add(desc_cmd_sett);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
{
bool r = command_line::handle_error_helper(desc_options, [&]() {
auto parser = po::command_line_parser(argc, argv).options(desc_options);
po::store(parser.run(), vm);
po::notify(vm);
@ -142,8 +135,7 @@ int main(int argc, char* argv[])
if (!r)
return 1;
if (command_line::get_arg(vm, command_line::arg_help))
{
if (command_line::get_arg(vm, command_line::arg_help)) {
std::cout << "Oxen '" << OXEN_RELEASE_NAME << "' (v" << OXEN_VERSION_FULL << ")\n\n";
std::cout << desc_options << std::endl;
return 1;
@ -159,7 +151,9 @@ int main(int argc, char* argv[])
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
network_type net_type = opt_testnet ? network_type::TESTNET : opt_devnet ? network_type::DEVNET : network_type::MAINNET;
network_type net_type = opt_testnet ? network_type::TESTNET
: opt_devnet ? network_type::DEVNET
: network_type::MAINNET;
bool opt_verbose = command_line::get_arg(vm, arg_verbose);
bool opt_dry_run = command_line::get_arg(vm, arg_dry_run);
@ -169,21 +163,18 @@ int main(int argc, char* argv[])
blockchain_objects_t blockchain_objects = {};
Blockchain* core_storage = &blockchain_objects.m_blockchain;
BlockchainDB* db = new_db();
if (db == NULL)
{
if (db == NULL) {
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
const fs::path filename = fs::u8path(command_line::get_arg(vm, cryptonote::arg_data_dir)) / db->get_db_name();
const fs::path filename =
fs::u8path(command_line::get_arg(vm, cryptonote::arg_data_dir)) / db->get_db_name();
log::warning(logcat, "Loading blockchain from folder {} ...", filename);
try
{
try {
db->open(filename, core_storage->nettype(), 0);
}
catch (const std::exception& e)
{
} catch (const std::exception& e) {
log::warning(logcat, "Error opening database: {}", e.what());
return 1;
}
@ -193,19 +184,20 @@ int main(int argc, char* argv[])
log::warning(logcat, "Source blockchain storage initialized OK");
std::map<uint64_t, uint64_t> known_spent_outputs;
if (input.empty())
{
if (input.empty()) {
std::map<uint64_t, std::pair<uint64_t, uint64_t>> outputs;
log::warning(logcat, "Scanning for known spent data...");
db->for_all_transactions([&](const crypto::hash &txid, const cryptonote::transaction &tx){
const bool miner_tx = tx.vin.size() == 1 && std::holds_alternative<txin_gen>(tx.vin[0]);
db->for_all_transactions(
[&](const crypto::hash& txid, const cryptonote::transaction& tx) {
const bool miner_tx =
tx.vin.size() == 1 && std::holds_alternative<txin_gen>(tx.vin[0]);
for (const auto& in : tx.vin)
if (const auto* txin = std::get_if<txin_to_key>(&in); txin && txin->amount != 0)
if (const auto* txin = std::get_if<txin_to_key>(&in);
txin && txin->amount != 0)
outputs[txin->amount].second++;
for (const auto &out: tx.vout)
{
for (const auto& out : tx.vout) {
uint64_t amount = out.amount;
if (miner_tx && tx.version >= txversion::v2_ringct)
amount = 0;
@ -217,15 +209,13 @@ int main(int argc, char* argv[])
outputs[amount].first++;
}
return true;
}, true);
},
true);
for (const auto &i: outputs)
{
for (const auto& i : outputs) {
known_spent_outputs[i.first] = i.second.second;
}
}
else
{
} else {
log::warning(logcat, "Loading known spent data...");
known_spent_outputs = load_outputs(input);
}
@ -233,22 +223,20 @@ int main(int argc, char* argv[])
log::warning(logcat, "Pruning known spent data...");
bool stop_requested = false;
tools::signal_handler::install([&stop_requested](int type) {
stop_requested = true;
});
tools::signal_handler::install([&stop_requested](int type) { stop_requested = true; });
db->batch_start();
size_t num_total_outputs = 0, num_prunable_outputs = 0, num_known_spent_outputs = 0, num_eligible_outputs = 0, num_eligible_known_spent_outputs = 0;
for (auto i = known_spent_outputs.begin(); i != known_spent_outputs.end(); ++i)
{
size_t num_total_outputs = 0, num_prunable_outputs = 0, num_known_spent_outputs = 0,
num_eligible_outputs = 0, num_eligible_known_spent_outputs = 0;
for (auto i = known_spent_outputs.begin(); i != known_spent_outputs.end(); ++i) {
uint64_t num_outputs = db->get_num_outputs(i->first);
num_total_outputs += num_outputs;
num_known_spent_outputs += i->second;
if (i->first == 0)
{
if (i->first == 0) {
if (opt_verbose)
log::info(logcat, "Ignoring output value {}, with {} outputs", i->first, num_outputs);
log::info(
logcat, "Ignoring output value {}, with {} outputs", i->first, num_outputs);
continue;
}
num_eligible_outputs += num_outputs;
@ -257,9 +245,11 @@ int main(int argc, char* argv[])
log::info(logcat, "{}: {}/{}", i->first, i->second, num_outputs);
if (num_outputs > i->second)
continue;
if (num_outputs && num_outputs < i->second)
{
log::error(logcat, "More outputs are spent than known for amount {}, not touching", i->first);
if (num_outputs && num_outputs < i->second) {
log::error(
logcat,
"More outputs are spent than known for amount {}, not touching",
i->first);
continue;
}
if (opt_verbose)

118
src/blockchain_utilities/blockchain_stats.cpp
View file

@ -26,27 +26,28 @@
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <date/date.h>
#include <boost/algorithm/string.hpp>
#include <chrono>
#include "blockchain_db/blockchain_db.h"
#include "blockchain_objects.h"
#include "common/command_line.h"
#include "common/varint.h"
#include "common/signal_handler.h"
#include "common/fs-format.h"
#include "common/signal_handler.h"
#include "common/varint.h"
#include "cryptonote_basic/cryptonote_boost_serialization.h"
#include "cryptonote_core/cryptonote_core.h"
#include "blockchain_objects.h"
#include "blockchain_db/blockchain_db.h"
#include "version.h"
#include "cryptonote_core/uptime_proof.h"
#include <date/date.h>
#include "version.h"
namespace po = boost::program_options;
using namespace cryptonote;
static bool stop_requested = false;
int main(int argc, char* argv[])
{
int main(int argc, char* argv[]) {
static auto logcat = log::Cat("bcutil");
TRY_ENTRY();
@ -59,14 +60,22 @@ int main(int argc, char* argv[])
auto opt_size = command_line::boost_option_sizes();
po::options_description desc_cmd_only("Command line options", opt_size.first, opt_size.second);
po::options_description desc_cmd_sett("Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<uint64_t> arg_block_start = {"block-start", "start at block number", block_start};
const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop};
const command_line::arg_descriptor<bool> arg_inputs = {"with-inputs", "with input stats", false};
const command_line::arg_descriptor<bool> arg_outputs = {"with-outputs", "with output stats", false};
const command_line::arg_descriptor<bool> arg_ringsize = {"with-ringsize", "with ringsize stats", false};
const command_line::arg_descriptor<bool> arg_hours = {"with-hours", "with txns per hour", false};
po::options_description desc_cmd_sett(
"Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_log_level = {
"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<uint64_t> arg_block_start = {
"block-start", "start at block number", block_start};
const command_line::arg_descriptor<uint64_t> arg_block_stop = {
"block-stop", "Stop at block number", block_stop};
const command_line::arg_descriptor<bool> arg_inputs = {
"with-inputs", "with input stats", false};
const command_line::arg_descriptor<bool> arg_outputs = {
"with-outputs", "with output stats", false};
const command_line::arg_descriptor<bool> arg_ringsize = {
"with-ringsize", "with ringsize stats", false};
const command_line::arg_descriptor<bool> arg_hours = {
"with-hours", "with txns per hour", false};
command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
@ -84,8 +93,7 @@ int main(int argc, char* argv[])
desc_options.add(desc_cmd_only).add(desc_cmd_sett);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
{
bool r = command_line::handle_error_helper(desc_options, [&]() {
auto parser = po::command_line_parser(argc, argv).options(desc_options);
po::store(parser.run(), vm);
po::notify(vm);
@ -94,8 +102,7 @@ int main(int argc, char* argv[])
if (!r)
return 1;
if (command_line::get_arg(vm, command_line::arg_help))
{
if (command_line::get_arg(vm, command_line::arg_help)) {
std::cout << "Oxen '" << OXEN_RELEASE_NAME << "' (v" << OXEN_VERSION_FULL << ")\n\n";
std::cout << desc_options << std::endl;
return 1;
@ -107,7 +114,8 @@ int main(int argc, char* argv[])
if (auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str()
<< std::endl;
throw std::runtime_error{"Incorrect log level"};
}
oxen::logging::init(log_file_path, log_level);
@ -116,7 +124,9 @@ int main(int argc, char* argv[])
std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir);
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
network_type net_type = opt_testnet ? network_type::TESTNET : opt_devnet ? network_type::DEVNET : network_type::MAINNET;
network_type net_type = opt_testnet ? network_type::TESTNET
: opt_devnet ? network_type::DEVNET
: network_type::MAINNET;
block_start = command_line::get_arg(vm, arg_block_start);
block_stop = command_line::get_arg(vm, arg_block_stop);
bool do_inputs = command_line::get_arg(vm, arg_inputs);
@ -128,8 +138,7 @@ int main(int argc, char* argv[])
blockchain_objects_t blockchain_objects = {};
Blockchain* core_storage = &blockchain_objects.m_blockchain;
BlockchainDB* db = new_db();
if (db == NULL)
{
if (db == NULL) {
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
@ -137,12 +146,9 @@ int main(int argc, char* argv[])
const fs::path filename = fs::u8path(opt_data_dir) / db->get_db_name();
log::warning(logcat, "Loading blockchain from folder {} ...", filename);
try
{
try {
db->open(filename, core_storage->nettype(), DBF_RDONLY);
}
catch (const std::exception& e)
{
} catch (const std::exception& e) {
log::warning(logcat, "Error opening database: {}", e.what());
return 1;
}
@ -151,9 +157,7 @@ int main(int argc, char* argv[])
CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
log::warning(logcat, "Source blockchain storage initialized OK");
tools::signal_handler::install([](int type) {
stop_requested = true;
});
tools::signal_handler::install([](int type) { stop_requested = true; });
const uint64_t db_height = db->height();
if (!block_stop)
@ -173,7 +177,8 @@ set y2range [0:*]
set ylabel "Txs/Day"
set y2label "Bytes"
set y2tics nomirror
plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' using (timecolumn(1,"%Y-%m-%d")):7 axes x1y2 with lines
plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' using
(timecolumn(1,"%Y-%m-%d")):7 axes x1y2 with lines
*/
// spit out a comment that GnuPlot can use as an index
@ -207,12 +212,10 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
uint32_t txhr[24] = {0};
unsigned int i;
for (uint64_t h = block_start; h < block_stop; ++h)
{
for (uint64_t h = block_start; h < block_stop; ++h) {
std::string bd = db->get_block_blob_from_height(h);
cryptonote::block blk;
if (!cryptonote::parse_and_validate_block_from_blob(bd, blk))
{
if (!cryptonote::parse_and_validate_block_from_blob(bd, blk)) {
log::warning(logcat, "Bad block from db");
return 1;
}
@ -223,13 +226,15 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
prev_ts = ts;
year_month_day prev_date{floor<days>(*prev_ts)};
// catch change of day
if (curr_date.day() > prev_date.day() || (curr_date.day() == day{1} && prev_date.day() > day{27}))
{
if (curr_date.day() > prev_date.day() ||
(curr_date.day() == day{1} && prev_date.day() > day{27})) {
// check for timestamp fudging around month ends
if (curr_date.day() == day{1} && prev_date.day() > day{27})
goto skip;
prev_ts = ts;
std::cout << format("%Y-%m-%d", prev_date) << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz;
std::cout << format("%Y-%m-%d", prev_date) << "\t" << currblks << "\t" << h << "\t"
<< currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t"
<< prevsz + currsz;
prevsz += currsz;
currsz = 0;
currblks = 0;
@ -238,16 +243,25 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
if (!tottxs)
tottxs = 1;
if (do_inputs) {
std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins / tottxs;
minins = 10; maxins = 0; totins = 0;
std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t"
<< totins / tottxs;
minins = 10;
maxins = 0;
totins = 0;
}
if (do_outputs) {
std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts / tottxs;
minouts = 10; maxouts = 0; totouts = 0;
std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t"
<< totouts / tottxs;
minouts = 10;
maxouts = 0;
totouts = 0;
}
if (do_ringsize) {
std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings / tottxs;
minrings = 50; maxrings = 0; totrings = 0;
std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t"
<< totrings / tottxs;
minrings = 50;
maxrings = 0;
totrings = 0;
}
tottxs = 0;
if (do_hours) {
@ -260,19 +274,15 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
}
skip:
currsz += bd.size();
for (const auto& tx_id : blk.tx_hashes)
{
if (!tx_id)
{
for (const auto& tx_id : blk.tx_hashes) {
if (!tx_id) {
throw std::runtime_error("Aborting: null txid");
}
if (!db->get_pruned_tx_blob(tx_id, bd))
{
if (!db->get_pruned_tx_blob(tx_id, bd)) {
throw std::runtime_error("Aborting: tx not found");
}
transaction tx;
if (!parse_and_validate_tx_base_from_blob(bd, tx))
{
if (!parse_and_validate_tx_base_from_blob(bd, tx)) {
log::warning(logcat, "Bad txn from db");
return 1;
}

124
src/blockchain_utilities/blockchain_usage.cpp
View file

@ -27,12 +27,12 @@
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "common/command_line.h"
#include "common/varint.h"
#include "common/fs-format.h"
#include "cryptonote_core/cryptonote_core.h"
#include "blockchain_objects.h"
#include "blockchain_db/blockchain_db.h"
#include "blockchain_objects.h"
#include "common/command_line.h"
#include "common/fs-format.h"
#include "common/varint.h"
#include "cryptonote_core/cryptonote_core.h"
#include "cryptonote_core/uptime_proof.h"
#include "version.h"
@ -41,40 +41,41 @@ using namespace cryptonote;
static auto logcat = log::Cat("quorum_cop");
struct output_data
{
struct output_data {
uint64_t amount;
uint64_t index;
mutable bool coinbase;
mutable uint64_t height;
output_data(uint64_t a, uint64_t i, bool cb, uint64_t h): amount(a), index(i), coinbase(cb), height(h) {}
bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; }
void info(bool c, uint64_t h) const { coinbase = c; height =h; }
output_data(uint64_t a, uint64_t i, bool cb, uint64_t h) :
amount(a), index(i), coinbase(cb), height(h) {}
bool operator==(const output_data& other) const {
return other.amount == amount && other.index == index;
}
void info(bool c, uint64_t h) const {
coinbase = c;
height = h;
}
};
namespace std
{
template<> struct hash<output_data>
{
size_t operator()(const output_data &od) const
{
namespace std {
template <>
struct hash<output_data> {
size_t operator()(const output_data& od) const {
const uint64_t data[2] = {od.amount, od.index};
crypto::hash h;
crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h);
return reinterpret_cast<const std::size_t&>(h);
}
};
}
} // namespace std
struct reference
{
struct reference {
uint64_t height;
uint64_t ring_size;
uint64_t position;
reference(uint64_t h, uint64_t rs, uint64_t p) : height(h), ring_size(rs), position(p) {}
};
int main(int argc, char* argv[])
{
int main(int argc, char* argv[]) {
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
@ -82,9 +83,12 @@ int main(int argc, char* argv[])
auto opt_size = command_line::boost_option_sizes();
po::options_description desc_cmd_only("Command line options", opt_size.first, opt_size.second);
po::options_description desc_cmd_sett("Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false};
po::options_description desc_cmd_sett(
"Command line options and settings options", opt_size.first, opt_size.second);
const command_line::arg_descriptor<std::string> arg_log_level = {
"log-level", "0-4 or categories", ""};
const command_line::arg_descriptor<bool> arg_rct_only = {
"rct-only", "Only work on ringCT outputs", false};
const command_line::arg_descriptor<std::string> arg_input = {"input", ""};
command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
@ -101,9 +105,10 @@ int main(int argc, char* argv[])
positional_options.add(arg_input.name, -1);
po::variables_map vm;
bool r = command_line::handle_error_helper(desc_options, [&]()
{
auto parser = po::command_line_parser(argc, argv).options(desc_options).positional(positional_options);
bool r = command_line::handle_error_helper(desc_options, [&]() {
auto parser = po::command_line_parser(argc, argv)
.options(desc_options)
.positional(positional_options);
po::store(parser.run(), vm);
po::notify(vm);
return true;
@ -111,8 +116,7 @@ int main(int argc, char* argv[])
if (!r)
return 1;
if (command_line::get_arg(vm, command_line::arg_help))
{
if (command_line::get_arg(vm, command_line::arg_help)) {
std::cout << "Oxen '" << OXEN_RELEASE_NAME << "' (v" << OXEN_VERSION_FULL << ")\n\n";
std::cout << desc_options << std::endl;
return 1;
@ -124,7 +128,8 @@ int main(int argc, char* argv[])
if (auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str()
<< std::endl;
throw std::runtime_error{"Incorrect log level"};
}
oxen::logging::init(log_file_path, log_level);
@ -132,14 +137,17 @@ int main(int argc, char* argv[])
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
network_type net_type = opt_testnet ? network_type::TESTNET : opt_devnet ? network_type::DEVNET : network_type::MAINNET;
network_type net_type = opt_testnet ? network_type::TESTNET
: opt_devnet ? network_type::DEVNET
: network_type::MAINNET;
bool opt_rct_only = command_line::get_arg(vm, arg_rct_only);
// If we wanted to use the memory pool, we would set up a fake_core.
// Use Blockchain instead of lower-level BlockchainDB for two reasons:
// 1. Blockchain has the init() method for easy setup
// 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash()
// 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(),
// get_block_by_hash()
//
// cannot match blockchain_storage setup above with just one line,
// e.g.
@ -152,8 +160,7 @@ int main(int argc, char* argv[])
Blockchain* core_storage = &blockchain_objects.m_blockchain;
tx_memory_pool& m_mempool = blockchain_objects.m_mempool;
BlockchainDB* db = new_db();
if (db == NULL)
{
if (db == NULL) {
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
@ -162,12 +169,9 @@ int main(int argc, char* argv[])
const fs::path filename = fs::u8path(input);
log::warning(logcat, "Loading blockchain from folder {} ...", filename);
try
{
try {
db->open(filename, core_storage->nettype(), DBF_RDONLY);
}
catch (const std::exception& e)
{
} catch (const std::exception& e) {
log::warning(logcat, "Error opening database: {}", e.what());
return 1;
}
@ -184,14 +188,14 @@ int main(int argc, char* argv[])
std::unordered_map<uint64_t, uint64_t> indices;
log::warning(logcat, "Reading blockchain from {}", input);
core_storage->for_all_transactions([&](const crypto::hash &hash, const cryptonote::transaction &tx)->bool
{
const bool coinbase = tx.vin.size() == 1 && std::holds_alternative<txin_gen>(tx.vin[0]);
core_storage->for_all_transactions(
[&](const crypto::hash& hash, const cryptonote::transaction& tx) -> bool {
const bool coinbase =
tx.vin.size() == 1 && std::holds_alternative<txin_gen>(tx.vin[0]);
const uint64_t height = core_storage->get_db().get_tx_block_height(hash);
// create new outputs
for (const auto &out: tx.vout)
{
for (const auto& out : tx.vout) {
if (opt_rct_only && out.amount)
continue;
uint64_t index = indices[out.amount]++;
@ -200,39 +204,39 @@ int main(int argc, char* argv[])
itb.first->first.info(coinbase, height);
}
for (const auto &in: tx.vin)
{
for (const auto& in : tx.vin) {
const auto* txin = std::get_if<txin_to_key>(&in);
if (!txin || (opt_rct_only && txin->amount != 0))
continue;
const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin->key_offsets);
for (size_t n = 0; n < txin->key_offsets.size(); ++n)
{
const std::vector<uint64_t> absolute =
cryptonote::relative_output_offsets_to_absolute(txin->key_offsets);
for (size_t n = 0; n < txin->key_offsets.size(); ++n) {
output_data od(txin->amount, absolute[n], coinbase, height);
outputs[od].push_back(reference(height, txin->key_offsets.size(), n));
}
}
return true;
}, true);
},
true);
std::unordered_map<uint64_t, uint64_t> counts;
size_t total = 0;
for (const auto &out: outputs)
{
for (const auto& out : outputs) {
counts[out.second.size()]++;
total++;
}
if (total > 0)
{
for (const auto &c: counts)
{
if (total > 0) {
for (const auto& c : counts) {
float percent = 100.f * c.second / total;
log::info(logcat, "{} outputs used {} times ({}%)", std::to_string(c.second), c.first, percent);
log::info(
logcat,
"{} outputs used {} times ({}%)",
std::to_string(c.second),
c.first,
percent);
}
}
else
{
} else {
log::info(logcat, "No outputs to process");
}

2
src/blockchain_utilities/blockchain_utilities.h
View file

@ -30,11 +30,9 @@
#include "version.h"
// bounds checking is done before writing to buffer, but buffer size
// should be a sensible maximum
#define BUFFER_SIZE (2 * 1024 * 1024)
#define CHUNK_SIZE_WARNING_THRESHOLD 500000
#define NUM_BLOCKS_PER_CHUNK 1
#define BLOCKCHAIN_RAW "blockchain.raw"

75
src/blockchain_utilities/blocksdat_file.cpp
View file

@ -28,36 +28,27 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "blocksdat_file.h"
#include "common/fs-format.h"
using namespace cryptonote;
namespace
{
namespace {
static auto logcat = log::Cat("bcutil");
std::string refresh_string = "\r \r";
}
} // namespace
bool BlocksdatFile::open_writer(const fs::path& file_path, uint64_t block_stop)
{
bool BlocksdatFile::open_writer(const fs::path& file_path, uint64_t block_stop) {
const fs::path dir_path = file_path.parent_path();
if (!dir_path.empty())
{
if (fs::exists(dir_path))
{
if (!fs::is_directory(dir_path))
{
if (!dir_path.empty()) {
if (fs::exists(dir_path)) {
if (!fs::is_directory(dir_path)) {
log::error(logcat, "export directory path is a file: {}", dir_path);
return false;
}
}
else
{
if (!fs::create_directory(dir_path))
{
} else {
if (!fs::create_directory(dir_path)) {
log::error(logcat, "Failed to create directory {}", dir_path);
return false;
}
@ -68,7 +59,8 @@ bool BlocksdatFile::open_writer(const fs::path& file_path, uint64_t block_stop)
log::info(logcat, "creating file");
m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
m_raw_data_file->open(
file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
if (m_raw_data_file->fail())
return false;
@ -77,9 +69,7 @@ bool BlocksdatFile::open_writer(const fs::path& file_path, uint64_t block_stop)
return true;
}
bool BlocksdatFile::initialize_file(uint64_t block_stop)
{
bool BlocksdatFile::initialize_file(uint64_t block_stop) {
const uint32_t nblocks = (block_stop + 1) / HASH_OF_HASHES_STEP;
unsigned char nblocksc[4];
@ -97,21 +87,20 @@ bool BlocksdatFile::initialize_file(uint64_t block_stop)
return true;
}
void BlocksdatFile::write_block(const crypto::hash& block_hash)
{
void BlocksdatFile::write_block(const crypto::hash& block_hash) {
m_hashes.push_back(block_hash);
while (m_hashes.size() >= HASH_OF_HASHES_STEP)
{
while (m_hashes.size() >= HASH_OF_HASHES_STEP) {
crypto::hash hash;
crypto::cn_fast_hash(m_hashes.data(), HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash);
memmove(m_hashes.data(), m_hashes.data() + HASH_OF_HASHES_STEP, (m_hashes.size() - HASH_OF_HASHES_STEP) * sizeof(crypto::hash));
memmove(m_hashes.data(),
m_hashes.data() + HASH_OF_HASHES_STEP,
(m_hashes.size() - HASH_OF_HASHES_STEP) * sizeof(crypto::hash));
m_hashes.resize(m_hashes.size() - HASH_OF_HASHES_STEP);
m_raw_data_file->write(reinterpret_cast<const char*>(hash.data()), hash.size());
}
}
bool BlocksdatFile::close()
{
bool BlocksdatFile::close() {
if (m_raw_data_file->fail())
return false;
@ -120,9 +109,11 @@ bool BlocksdatFile::close()
return true;
}
bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, fs::path& output_file, uint64_t requested_block_stop)
{
bool BlocksdatFile::store_blockchain_raw(
Blockchain* _blockchain_storage,
tx_memory_pool* _tx_pool,
fs::path& output_file,
uint64_t requested_block_stop) {
uint64_t num_blocks_written = 0;
m_blockchain_storage = _blockchain_storage;
uint64_t progress_interval = 100;
@ -130,25 +121,24 @@ bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
uint64_t block_start = 0;
uint64_t block_stop = 0;
log::info(logcat, "source blockchain height: {}", m_blockchain_storage->get_current_blockchain_height()-1);
if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height()))
{
log::info(
logcat,
"source blockchain height: {}",
m_blockchain_storage->get_current_blockchain_height() - 1);
if ((requested_block_stop > 0) &&
(requested_block_stop < m_blockchain_storage->get_current_blockchain_height())) {
log::info(logcat, "Using requested block height: {}", requested_block_stop);
block_stop = requested_block_stop;
}
else
{
} else {
block_stop = m_blockchain_storage->get_current_blockchain_height() - 1;
log::info(logcat, "Using block height of source blockchain: {}", block_stop);
}
log::info(logcat, "Storing blocks raw data...");
if (!BlocksdatFile::open_writer(output_file, block_stop))
{
if (!BlocksdatFile::open_writer(output_file, block_stop)) {
log::error(logcat, "failed to open raw file for write");
return false;
}
for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
{
for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height) {
// this method's height refers to 0-based height (genesis block = height 0)
crypto::hash hash = m_blockchain_storage->get_block_id_by_height(m_cur_height);
write_block(hash);
@ -168,4 +158,3 @@ bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
return BlocksdatFile::close();
}

40
src/blockchain_utilities/blocksdat_file.h
View file

@ -28,41 +28,36 @@
#pragma once
#include <boost/iostreams/stream_buffer.hpp>
#include <boost/iostreams/stream.hpp>
#include <algorithm>
#include <atomic>
#include <boost/iostreams/copy.hpp>
#include <boost/iostreams/device/back_inserter.hpp>
#include <boost/iostreams/filtering_streambuf.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/iostreams/stream_buffer.hpp>
#include <cstdio>
#include <fstream>
#include "blockchain_db/blockchain_db.h"
#include "blockchain_utilities.h"
#include "common/command_line.h"
#include "common/fs.h"
#include "cryptonote_basic/cryptonote_basic.h"
#include "cryptonote_basic/cryptonote_boost_serialization.h"
#include "cryptonote_core/blockchain.h"
#include "blockchain_db/blockchain_db.h"
#include <algorithm>
#include <cstdio>
#include <fstream>
#include <boost/iostreams/copy.hpp>
#include <atomic>
#include "common/command_line.h"
#include "common/fs.h"
#include "version.h"
#include "blockchain_utilities.h"
using namespace cryptonote;
class BlocksdatFile
{
class BlocksdatFile {
public:
bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp,
fs::path& output_file, uint64_t use_block_height=0);
bool store_blockchain_raw(
cryptonote::Blockchain* cs,
cryptonote::tx_memory_pool* txp,
fs::path& output_file,
uint64_t use_block_height = 0);
protected:
Blockchain* m_blockchain_storage;
std::ofstream* m_raw_data_file;
@ -74,7 +69,6 @@ protected:
void write_block(const crypto::hash& block_hash);
private:
uint64_t m_cur_height; // tracks current height during export
std::vector<crypto::hash> m_hashes;
};

236
src/blockchain_utilities/bootstrap_file.cpp
View file

@ -27,17 +27,15 @@
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "bootstrap_serialization.h"
#include "serialization/binary_utils.h" // dump_binary(), parse_binary()
#include "bootstrap_file.h"
#include "bootstrap_serialization.h"
#include "common/fs-format.h"
#include "serialization/binary_utils.h" // dump_binary(), parse_binary()
using namespace cryptonote;
namespace
{
namespace {
// This number was picked by taking the leading 4 bytes from this output:
// echo Oxen bootstrap file | sha1sum
const uint32_t blockchain_raw_magic = 0x28721586;
@ -45,27 +43,18 @@ namespace
std::string refresh_string = "\r \r";
auto logcat = log::Cat("bcutil");
}
} // namespace
bool BootstrapFile::open_writer(const fs::path& file_path)
{
bool BootstrapFile::open_writer(const fs::path& file_path) {
const auto dir_path = file_path.parent_path();
if (!dir_path.empty())
{
if (fs::exists(dir_path))
{
if (!fs::is_directory(dir_path))
{
if (!dir_path.empty()) {
if (fs::exists(dir_path)) {
if (!fs::is_directory(dir_path)) {
log::error(logcat, "export directory path is a file: {}", dir_path);
return false;
}
}
else
{
if (!fs::create_directory(dir_path))
{
} else {
if (!fs::create_directory(dir_path)) {
log::error(logcat, "Failed to create directory {}", dir_path);
return false;
}
@ -77,28 +66,34 @@ bool BootstrapFile::open_writer(const fs::path& file_path)
bool do_initialize_file = false;
uint64_t num_blocks = 0;
if (! fs::exists(file_path))
{
if (!fs::exists(file_path)) {
log::debug(logcat, "creating file");
do_initialize_file = true;
num_blocks = 0;
}
else
{
} else {
num_blocks = count_blocks(file_path.string());
log::debug(logcat, "appending to existing file with height: {} total blocks: {}", num_blocks-1, num_blocks);
log::debug(
logcat,
"appending to existing file with height: {} total blocks: {}",
num_blocks - 1,
num_blocks);
}
m_height = num_blocks;
if (do_initialize_file)
m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
m_raw_data_file->open(
file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
else
m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::app | std::ios::ate);
m_raw_data_file->open(
file_path.string(),
std::ios_base::binary | std::ios_base::out | std::ios::app | std::ios::ate);
if (m_raw_data_file->fail())
return false;
m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer);
m_output_stream =
new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(
m_buffer);
if (m_output_stream == nullptr)
return false;
@ -108,9 +103,7 @@ bool BootstrapFile::open_writer(const fs::path& file_path)
return true;
}
bool BootstrapFile::initialize_file()
{
bool BootstrapFile::initialize_file() {
const uint32_t file_magic = blockchain_raw_magic;
std::string blob;
@ -132,7 +125,8 @@ bool BootstrapFile::initialize_file()
bbi.block_last_pos = 0;
buffer_type buffer2;
boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>> output_stream_header(buffer2);
boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>
output_stream_header(buffer2);
uint32_t bd_size = 0;
@ -143,7 +137,8 @@ bool BootstrapFile::initialize_file()
try {
blob = serialization::dump_binary(bd_size);
} catch (const std::exception& e) {
throw std::runtime_error("Error in serialization of bootstrap::file_info size: "s + e.what());
throw std::runtime_error(
"Error in serialization of bootstrap::file_info size: "s + e.what());
}
output_stream_header << blob;
output_stream_header << bd;
@ -155,27 +150,27 @@ bool BootstrapFile::initialize_file()
try {
blob = serialization::dump_binary(bd_size);
} catch (const std::exception& e) {
throw std::runtime_error("Error in serialization of bootstrap::blocks_info size: "s + e.what());
throw std::runtime_error(
"Error in serialization of bootstrap::blocks_info size: "s + e.what());
}
output_stream_header << blob;
output_stream_header << bd;
output_stream_header.flush();
output_stream_header << std::string(header_size-buffer2.size(), 0); // fill in rest with null bytes
output_stream_header << std::string(
header_size - buffer2.size(), 0); // fill in rest with null bytes
output_stream_header.flush();
std::copy(buffer2.begin(), buffer2.end(), std::ostreambuf_iterator<char>(*m_raw_data_file));
return true;
}
void BootstrapFile::flush_chunk()
{
void BootstrapFile::flush_chunk() {
m_output_stream->flush();
uint32_t chunk_size = m_buffer.size();
// log::trace(logcat, "chunk_size {}", chunk_size);
if (chunk_size > BUFFER_SIZE)
{
if (chunk_size > BUFFER_SIZE) {
log::warning(logcat, "WARNING: chunk_size {} > BUFFER_SIZE {}", chunk_size, BUFFER_SIZE);
}
@ -187,8 +182,7 @@ void BootstrapFile::flush_chunk()
}
*m_raw_data_file << blob;
if (m_max_chunk < chunk_size)
{
if (m_max_chunk < chunk_size) {
m_max_chunk = chunk_size;
}
long pos_before = m_raw_data_file->tellp();
@ -196,20 +190,25 @@ void BootstrapFile::flush_chunk()
m_raw_data_file->flush();
long pos_after = m_raw_data_file->tellp();
long num_chars_written = pos_after - pos_before;
if (static_cast<unsigned long>(num_chars_written) != chunk_size)
{
log::error(logcat, "Error writing chunk: height: {} chunk_size: {} num chars written: {}", m_cur_height, chunk_size, num_chars_written);
if (static_cast<unsigned long>(num_chars_written) != chunk_size) {
log::error(
logcat,
"Error writing chunk: height: {} chunk_size: {} num chars written: {}",
m_cur_height,
chunk_size,
num_chars_written);
throw std::runtime_error("Error writing chunk");
}
m_buffer.clear();
delete m_output_stream;
m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer);
m_output_stream =
new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(
m_buffer);
log::debug(logcat, "flushed chunk: chunk_size: {}", chunk_size);
}
void BootstrapFile::write_block(block& block)
{
void BootstrapFile::write_block(block& block) {
bootstrap::block_package bp;
bp.block = block;
@ -217,12 +216,9 @@ void BootstrapFile::write_block(block& block)
uint64_t block_height = var::get<txin_gen>(block.miner_tx.vin.front()).height;
// now add all regular transactions
for (const auto& tx_id : block.tx_hashes)
{
if (!tx_id)
{
for (const auto& tx_id : block.tx_hashes) {
if (!tx_id) {
throw std::runtime_error("Aborting: null txid");
}
transaction tx = m_blockchain_storage->get_db().get_tx(tx_id);
@ -233,13 +229,15 @@ void BootstrapFile::write_block(block& block)
// these non-coinbase txs will be serialized using this structure
bp.txs = txs;
// These three attributes are currently necessary for a fast import that adds blocks without verification.
// These three attributes are currently necessary for a fast import that adds blocks without
// verification.
bool include_extra_block_data = true;
if (include_extra_block_data)
{
if (include_extra_block_data) {
size_t block_weight = m_blockchain_storage->get_db().get_block_weight(block_height);
difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height);
uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height);
difficulty_type cumulative_difficulty =
m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height);
uint64_t coins_generated =
m_blockchain_storage->get_db().get_block_already_generated_coins(block_height);
bp.block_weight = block_weight;
bp.cumulative_difficulty = cumulative_difficulty;
@ -250,8 +248,7 @@ void BootstrapFile::write_block(block& block)
m_output_stream->write((const char*)bd.data(), bd.size());
}
bool BootstrapFile::close()
{
bool BootstrapFile::close() {
if (m_raw_data_file->fail())
return false;
@ -261,17 +258,18 @@ bool BootstrapFile::close()
return true;
}
bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, fs::path& output_file, uint64_t requested_block_stop)
{
bool BootstrapFile::store_blockchain_raw(
Blockchain* _blockchain_storage,
tx_memory_pool* _tx_pool,
fs::path& output_file,
uint64_t requested_block_stop) {
uint64_t num_blocks_written = 0;
m_max_chunk = 0;
m_blockchain_storage = _blockchain_storage;
m_tx_pool = _tx_pool;
uint64_t progress_interval = 100;
log::info(logcat, "Storing blocks raw data...");
if (!BootstrapFile::open_writer(output_file))
{
if (!BootstrapFile::open_writer(output_file)) {
log::error(logcat, "failed to open raw file for write");
return false;
}
@ -282,19 +280,19 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
// height.
uint64_t block_start = m_height;
uint64_t block_stop = 0;
log::info(logcat, "source blockchain height: {}", m_blockchain_storage->get_current_blockchain_height()-1);
if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height()))
{
log::info(
logcat,
"source blockchain height: {}",
m_blockchain_storage->get_current_blockchain_height() - 1);
if ((requested_block_stop > 0) &&
(requested_block_stop < m_blockchain_storage->get_current_blockchain_height())) {
log::info(logcat, "Using requested block height: {}", requested_block_stop);
block_stop = requested_block_stop;
}
else
{
} else {
block_stop = m_blockchain_storage->get_current_blockchain_height() - 1;
log::info(logcat, "Using block height of source blockchain: {}", block_stop);
}
for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
{
for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height) {
// this method's height refers to 0-based height (genesis block = height 0)
crypto::hash hash = m_blockchain_storage->get_block_id_by_height(m_cur_height);
m_blockchain_storage->get_block_by_hash(hash, b);
@ -308,9 +306,9 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
std::cout << "block " << m_cur_height << "/" << block_stop << "\r" << std::flush;
}
}
// NOTE: use of NUM_BLOCKS_PER_CHUNK is a placeholder in case multi-block chunks are later supported.
if (m_cur_height % NUM_BLOCKS_PER_CHUNK != 0)
{
// NOTE: use of NUM_BLOCKS_PER_CHUNK is a placeholder in case multi-block chunks are later
// supported.
if (m_cur_height % NUM_BLOCKS_PER_CHUNK != 0) {
flush_chunk();
}
// print message for last block, which may not have been printed yet due to progress_interval
@ -324,8 +322,7 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
return BootstrapFile::close();
}
uint64_t BootstrapFile::seek_to_first_chunk(fs::ifstream& import_file)
{
uint64_t BootstrapFile::seek_to_first_chunk(fs::ifstream& import_file) {
uint32_t file_magic;
std::string str1;
@ -341,12 +338,10 @@ uint64_t BootstrapFile::seek_to_first_chunk(fs::ifstream& import_file)
throw std::runtime_error("Error in deserialization of file_magic: "s + e.what());
}
if (file_magic != blockchain_raw_magic)
{
if (file_magic != blockchain_raw_magic) {
log::error(logcat, "bootstrap file not recognized");
throw std::runtime_error("Aborting");
}
else
} else
log::info(logcat, "bootstrap file recognized");
uint32_t buflen_file_info;
@ -374,7 +369,11 @@ uint64_t BootstrapFile::seek_to_first_chunk(fs::ifstream& import_file)
} catch (const std::exception& e) {
throw std::runtime_error("Error in deserialization of bootstrap::file_info: "s + e.what());
}
log::info(logcat, "bootstrap file v{}.{}", unsigned(bfi.major_version), unsigned(bfi.minor_version));
log::info(
logcat,
"bootstrap file v{}.{}",
unsigned(bfi.major_version),
unsigned(bfi.minor_version));
log::info(logcat, "bootstrap magic size: {}", sizeof(file_magic));
log::info(logcat, "bootstrap header size: {}", bfi.header_size);
@ -384,15 +383,14 @@ uint64_t BootstrapFile::seek_to_first_chunk(fs::ifstream& import_file)
return full_header_size;
}
uint64_t BootstrapFile::count_bytes(fs::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit)
{
uint64_t BootstrapFile::count_bytes(
fs::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit) {
uint64_t bytes_read = 0;
uint32_t chunk_size;
char buf1[sizeof(chunk_size)];
std::string str1;
h = 0;
while (1)
{
while (1) {
import_file.read(buf1, sizeof(chunk_size));
if (!import_file) {
std::cout << refresh_string;
@ -409,27 +407,45 @@ uint64_t BootstrapFile::count_bytes(fs::ifstream& import_file, uint64_t blocks,
}
log::debug(logcat, "chunk_size: {}", chunk_size);
if (chunk_size > BUFFER_SIZE)
{
if (chunk_size > BUFFER_SIZE) {
std::cout << refresh_string;
log::warning(logcat, "WARNING: chunk_size {} > BUFFER_SIZE {} height: {}, offset {}", chunk_size, BUFFER_SIZE, h-1, bytes_read);
log::warning(
logcat,
"WARNING: chunk_size {} > BUFFER_SIZE {} height: {}, offset {}",
chunk_size,
BUFFER_SIZE,
h - 1,
bytes_read);
throw std::runtime_error("Aborting: chunk size exceeds buffer size");
}
if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD)
{
if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD) {
std::cout << refresh_string;
log::debug(logcat, "NOTE: chunk_size {} > {} height: {}, offset {}", chunk_size, CHUNK_SIZE_WARNING_THRESHOLD, h-1, bytes_read);
}
else if (chunk_size <= 0) {
log::debug(
logcat,
"NOTE: chunk_size {} > {} height: {}, offset {}",
chunk_size,
CHUNK_SIZE_WARNING_THRESHOLD,
h - 1,
bytes_read);
} else if (chunk_size <= 0) {
std::cout << refresh_string;
log::debug(logcat, "ERROR: chunk_size {} <= 0 height: {}, offset {}", chunk_size, h-1, bytes_read);
log::debug(
logcat,
"ERROR: chunk_size {} <= 0 height: {}, offset {}",
chunk_size,
h - 1,
bytes_read);
throw std::runtime_error("Aborting");
}
// skip to next expected block size value
import_file.seekg(chunk_size, std::ios_base::cur);
if (!import_file) {
std::cout << refresh_string;
log::error(logcat, "ERROR: unexpected end of file: bytes read before error: {} of chunk_size {}", import_file.gcount(), chunk_size);
log::error(
logcat,
"ERROR: unexpected end of file: bytes read before error: {} of chunk_size {}",
import_file.gcount(),
chunk_size);
throw std::runtime_error("Aborting");
}
bytes_read += chunk_size;
@ -440,8 +456,7 @@ uint64_t BootstrapFile::count_bytes(fs::ifstream& import_file, uint64_t blocks,
return bytes_read;
}
uint64_t BootstrapFile::count_blocks(const fs::path& import_file_path)
{
uint64_t BootstrapFile::count_blocks(const fs::path& import_file_path) {
std::streampos dummy_pos;
uint64_t dummy_height = 0;
return count_blocks(import_file_path, dummy_pos, dummy_height);
@ -450,10 +465,9 @@ uint64_t BootstrapFile::count_blocks(const fs::path& import_file_path)
// If seek_height is non-zero on entry, return a stream position <= this height when finished.
// And return the actual height corresponding to this position. Allows the caller to locate its
// starting position without having to reread the entire file again.
uint64_t BootstrapFile::count_blocks(const fs::path& import_file_path, std::streampos &start_pos, uint64_t& seek_height)
{
if (std::error_code ec; !fs::exists(import_file_path, ec))
{
uint64_t BootstrapFile::count_blocks(
const fs::path& import_file_path, std::streampos& start_pos, uint64_t& seek_height) {
if (std::error_code ec; !fs::exists(import_file_path, ec)) {
log::error(logcat, "bootstrap file not found: {}", import_file_path);
throw std::runtime_error("Aborting");
}
@ -461,8 +475,7 @@ uint64_t BootstrapFile::count_blocks(const fs::path& import_file_path, std::stre
uint64_t start_height = seek_height;
uint64_t h = 0;
if (import_file.fail())
{
if (import_file.fail()) {
log::error(logcat, "import_file.open() fail");
throw std::runtime_error("Aborting");
}
@ -475,19 +488,16 @@ uint64_t BootstrapFile::count_blocks(const fs::path& import_file_path, std::stre
uint64_t bytes_read = 0, blocks;
int progress_interval = 10;
while (! quit)
{
if (start_height && h + progress_interval >= start_height - 1)
{
while (!quit) {
if (start_height && h + progress_interval >= start_height - 1) {
start_height = 0;
start_pos = import_file.tellg();
seek_height = h;
}
bytes_read += count_bytes(import_file, progress_interval, blocks, quit);
h += blocks;
std::cout << "\r" << "block height: " << h-1 <<
" \r" <<
std::flush;
std::cout << "\r"
<< "block height: " << h - 1 << " \r" << std::flush;
// std::cout << refresh_string;
log::debug(logcat, "Number bytes scanned: {}", bytes_read);

41
src/blockchain_utilities/bootstrap_file.h
View file

@ -28,44 +28,40 @@
#pragma once
#include <boost/iostreams/stream_buffer.hpp>
#include <boost/iostreams/stream.hpp>
#include <algorithm>
#include <atomic>
#include <boost/iostreams/copy.hpp>
#include <boost/iostreams/device/back_inserter.hpp>
#include <boost/iostreams/filtering_streambuf.hpp>
#include "cryptonote_basic/cryptonote_basic.h"
#include "cryptonote_core/blockchain.h"
#include <algorithm>
#include <boost/iostreams/stream.hpp>
#include <boost/iostreams/stream_buffer.hpp>
#include <cstdio>
#include <fstream>
#include <boost/iostreams/copy.hpp>
#include <atomic>
#include "common/command_line.h"
#include "common/fs.h"
#include "version.h"
#include "blockchain_utilities.h"
#include "common/command_line.h"
#include "common/fs.h"
#include "cryptonote_basic/cryptonote_basic.h"
#include "cryptonote_core/blockchain.h"
#include "version.h"
using namespace cryptonote;
class BootstrapFile
{
class BootstrapFile {
public:
uint64_t count_bytes(fs::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit);
uint64_t count_blocks(const fs::path& dir_path, std::streampos& start_pos, uint64_t& seek_height);
uint64_t count_blocks(
const fs::path& dir_path, std::streampos& start_pos, uint64_t& seek_height);
uint64_t count_blocks(const fs::path& dir_path);
uint64_t seek_to_first_chunk(fs::ifstream& import_file);
bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp,
fs::path& output_file, uint64_t use_block_height=0);
bool store_blockchain_raw(
cryptonote::Blockchain* cs,
cryptonote::tx_memory_pool* txp,
fs::path& output_file,
uint64_t use_block_height = 0);
protected:
Blockchain* m_blockchain_storage;
tx_memory_pool* m_tx_pool;
@ -82,7 +78,6 @@ protected:
void flush_chunk();
private:
uint64_t m_height;
uint64_t m_cur_height; // tracks current height during export
uint32_t m_max_chunk;

19
src/blockchain_utilities/bootstrap_serialization.h
View file

@ -31,14 +31,9 @@
#include "cryptonote_basic/cryptonote_boost_serialization.h"
#include "cryptonote_basic/difficulty.h"
namespace cryptonote { namespace bootstrap {
namespace cryptonote
{
namespace bootstrap
{
struct file_info
{
struct file_info {
uint8_t major_version;
uint8_t minor_version;
uint32_t header_size;
@ -50,8 +45,7 @@ namespace cryptonote
END_SERIALIZE()
};
struct blocks_info
{
struct blocks_info {
// block heights of file's first and last blocks, zero-based indexes
uint64_t block_first;
uint64_t block_last;
@ -66,8 +60,7 @@ namespace cryptonote
END_SERIALIZE()
};
struct block_package
{
struct block_package {
cryptonote::block block;
std::vector<transaction> txs;
size_t block_weight;
@ -83,6 +76,4 @@ namespace cryptonote
END_SERIALIZE()
};
}
}
}} // namespace cryptonote::bootstrap

115
src/blockchain_utilities/sn_key_tool.cpp
View file

@ -2,16 +2,18 @@
extern "C" {
#include <sodium.h>
}
#include <iostream>
#include <fstream>
#include <oxenc/hex.h>
#include <oxenc/base32z.h>
#include <string_view>
#include <string>
#include <list>
#include <oxenc/hex.h>
#include <array>
#include <cstring>
#include <fstream>
#include <iostream>
#include <list>
#include <optional>
#include <string>
#include <string_view>
#include "common/fs.h"
std::string_view arg0;
@ -91,7 +93,8 @@ std::array<unsigned char, crypto_core_ed25519_BYTES> pubkey_from_privkey(ustring
return pubkey;
}
template <size_t N, std::enable_if_t<(N >= 32), int> = 0>
std::array<unsigned char, crypto_core_ed25519_BYTES> pubkey_from_privkey(const std::array<unsigned char, N>& privkey) {
std::array<unsigned char, crypto_core_ed25519_BYTES> pubkey_from_privkey(
const std::array<unsigned char, N>& privkey) {
return pubkey_from_privkey(ustring_view{privkey.data(), 32});
}
@ -117,7 +120,11 @@ int generate(bool ed25519, std::list<std::string_view> args) {
overwrite = true;
if (!overwrite && fs::exists(fs::u8path(filename)))
return error(2, filename + " to generate already exists, pass `--overwrite' if you want to overwrite it");
return error(
2,
filename +
" to generate already exists, pass `--overwrite' if you want to overwrite "
"it");
std::array<unsigned char, crypto_sign_PUBLICKEYBYTES> pubkey;
std::array<unsigned char, crypto_sign_SECRETKEYBYTES> seckey;
@ -147,18 +154,20 @@ int generate(bool ed25519, std::list<std::string_view> args) {
out.write(reinterpret_cast<const char*>(privkey.data()), privkey.size());
if (!out.good())
return error(2, "Failed to write to output file '" + filename + "': " + std::strerror(errno));
return error(
2, "Failed to write to output file '" + filename + "': " + std::strerror(errno));
std::cout << "Generated SN " << (ed25519 ? "Ed25519 secret key" : "legacy private key") << " in " << filename << "\n";
std::cout << "Generated SN " << (ed25519 ? "Ed25519 secret key" : "legacy private key")
<< " in " << filename << "\n";
if (ed25519) {
std::array<unsigned char, crypto_scalarmult_curve25519_BYTES> x_pubkey;
if (0 != crypto_sign_ed25519_pk_to_curve25519(x_pubkey.data(), pubkey.data()))
return error(14, "Internal error: unable to convert Ed25519 pubkey to X25519 pubkey");
std::cout <<
"Public key: " << oxenc::to_hex(pubkey.begin(), pubkey.end()) <<
"\nX25519 pubkey: " << oxenc::to_hex(x_pubkey.begin(), x_pubkey.end()) <<
"\nLokinet address: " << oxenc::to_base32z(pubkey.begin(), pubkey.end()) << ".snode\n";
std::cout << "Public key: " << oxenc::to_hex(pubkey.begin(), pubkey.end())
<< "\nX25519 pubkey: " << oxenc::to_hex(x_pubkey.begin(), x_pubkey.end())
<< "\nLokinet address: " << oxenc::to_base32z(pubkey.begin(), pubkey.end())
<< ".snode\n";
} else {
std::cout << "Public key: " << oxenc::to_hex(pubkey.begin(), pubkey.end()) << "\n";
}
@ -204,10 +213,15 @@ int show(std::list<std::string_view> args) {
ed25519 = true;
}
if (!legacy && !ed25519)
return error(2, "Could not autodetect key type from " + std::to_string(size) + "-byte file; check the file or pass the --ed25519 or --legacy argument");
return error(
2,
"Could not autodetect key type from " + std::to_string(size) +
"-byte file; check the file or pass the --ed25519 or --legacy argument");
if (size < 32)
return error(2, "File size (" + std::to_string(size) + " bytes) is too small to be a secret key");
return error(
2,
"File size (" + std::to_string(size) + " bytes) is too small to be a secret key");
std::array<unsigned char, crypto_core_ed25519_BYTES> pubkey;
std::array<unsigned char, crypto_scalarmult_curve25519_BYTES> x_pubkey;
@ -219,9 +233,10 @@ int show(std::list<std::string_view> args) {
if (legacy) {
pubkey = pubkey_from_privkey(seckey);
std::cout << filename.u8string() << " (legacy SN keypair)" << "\n==========" <<
"\nPrivate key: " << oxenc::to_hex(seckey.begin(), seckey.begin() + 32) <<
"\nPublic key: " << oxenc::to_hex(pubkey.begin(), pubkey.end()) << "\n\n";
std::cout << filename.u8string() << " (legacy SN keypair)"
<< "\n=========="
<< "\nPrivate key: " << oxenc::to_hex(seckey.begin(), seckey.begin() + 32)
<< "\nPublic key: " << oxenc::to_hex(pubkey.begin(), pubkey.end()) << "\n\n";
return 0;
}
@ -233,17 +248,27 @@ int show(std::list<std::string_view> args) {
ustring_view privkey{privkey_signhash.data(), 32};
pubkey = pubkey_from_privkey(privkey);
if (size >= 64 && ustring_view{pubkey.data(), pubkey.size()} != ustring_view{seckey.data() + 32, 32})
return error(13, "Error: derived pubkey (" + oxenc::to_hex(pubkey.begin(), pubkey.end()) + ")"
" != embedded pubkey (" + oxenc::to_hex(seckey.begin() + 32, seckey.end()) + ")");
if (size >= 64 &&
ustring_view{pubkey.data(), pubkey.size()} != ustring_view{seckey.data() + 32, 32})
return error(
13,
"Error: derived pubkey (" + oxenc::to_hex(pubkey.begin(), pubkey.end()) +
")"
" != embedded pubkey (" +
oxenc::to_hex(seckey.begin() + 32, seckey.end()) + ")");
if (0 != crypto_sign_ed25519_pk_to_curve25519(x_pubkey.data(), pubkey.data()))
return error(14, "Unable to convert Ed25519 pubkey to X25519 pubkey; is this a really valid secret key?");
return error(
14,
"Unable to convert Ed25519 pubkey to X25519 pubkey; is this a really valid secret "
"key?");
std::cout << filename << " (Ed25519 SN keypair)" << "\n==========" <<
"\nSecret key: " << oxenc::to_hex(seckey.begin(), seckey.begin() + 32) <<
"\nPublic key: " << oxenc::to_hex(pubkey.begin(), pubkey.end()) <<
"\nX25519 pubkey: " << oxenc::to_hex(x_pubkey.begin(), x_pubkey.end()) <<
"\nLokinet address: " << oxenc::to_base32z(pubkey.begin(), pubkey.end()) << ".snode\n\n";
std::cout << filename << " (Ed25519 SN keypair)"
<< "\n=========="
<< "\nSecret key: " << oxenc::to_hex(seckey.begin(), seckey.begin() + 32)
<< "\nPublic key: " << oxenc::to_hex(pubkey.begin(), pubkey.end())
<< "\nX25519 pubkey: " << oxenc::to_hex(x_pubkey.begin(), x_pubkey.end())
<< "\nLokinet address: " << oxenc::to_base32z(pubkey.begin(), pubkey.end())
<< ".snode\n\n";
return 0;
}
@ -298,21 +323,31 @@ int restore(bool ed25519, std::list<std::string_view> args) {
if (ed25519) {
std::array<unsigned char, crypto_scalarmult_curve25519_BYTES> x_pubkey;
if (0 != crypto_sign_ed25519_pk_to_curve25519(x_pubkey.data(), pubkey.data()))
return error(14, "Unable to convert Ed25519 pubkey to X25519 pubkey; is this a really valid secret key?");
std::cout << "X25519 pubkey: " << oxenc::to_hex(x_pubkey.begin(), x_pubkey.end()) <<
"\nLokinet address: " << oxenc::to_base32z(pubkey.begin(), pubkey.end()) << ".snode";
return error(
14,
"Unable to convert Ed25519 pubkey to X25519 pubkey; is this a really valid "
"secret key?");
std::cout << "X25519 pubkey: " << oxenc::to_hex(x_pubkey.begin(), x_pubkey.end())
<< "\nLokinet address: " << oxenc::to_base32z(pubkey.begin(), pubkey.end())
<< ".snode";
}
if (pubkey_expected) {
if (*pubkey_expected != pubkey)
return error(2, "Derived pubkey (" + oxenc::to_hex(pubkey.begin(), pubkey.end()) + ") doesn't match "
"provided pubkey (" + oxenc::to_hex(pubkey_expected->begin(), pubkey_expected->end()) + ")");
return error(
2,
"Derived pubkey (" + oxenc::to_hex(pubkey.begin(), pubkey.end()) +
") doesn't match "
"provided pubkey (" +
oxenc::to_hex(pubkey_expected->begin(), pubkey_expected->end()) + ")");
} else {
if (ed25519 && filename.size() >= 4 && filename.substr(filename.size() - 4) == "/key") {
std::cout << "\n\n\x1b[31;1m"
"Warning: You are trying to restore a file named 'key' using the 'restore'\n"
"command, which is intended for the key_ed25519 key file; for old service nodes\n"
"Warning: You are trying to restore a file named 'key' using the "
"'restore'\n"
"command, which is intended for the key_ed25519 key file; for old service "
"nodes\n"
"with both key files you want to use 'restore-legacy' to restore the old\n"
"(pre-Loki 8.x) pubkey.\x1b[0m\n";
}
@ -328,7 +363,11 @@ int restore(bool ed25519, std::list<std::string_view> args) {
auto filepath = fs::u8path(filename);
if (!overwrite && fs::exists(filepath))
return error(2, filename + " to generate already exists, pass `--overwrite' if you want to overwrite it");
return error(
2,
filename +
" to generate already exists, pass `--overwrite' if you want to overwrite "
"it");
fs::ofstream out{filepath, std::ios::trunc | std::ios::binary};
if (!out.good())
@ -339,13 +378,13 @@ int restore(bool ed25519, std::list<std::string_view> args) {
out.write(reinterpret_cast<const char*>(seed.data()), seed.size());
if (!out.good())
return error(2, "Failed to write to output file '" + filename + "': " + std::strerror(errno));
return error(
2, "Failed to write to output file '" + filename + "': " + std::strerror(errno));
std::cout << "Saved secret key to " << filename << "\n";
return 0;
}
int main(int argc, char* argv[]) {
arg0 = argv[0];
if (argc < 2)

34
src/blocks/blocks.h
View file

@ -1,22 +1,30 @@
#pragma once
#include "cryptonote_config.h"
#include <string_view>
namespace blocks
{
#include "cryptonote_config.h"
template <cryptonote::network_type Network> std::string_view checkpoint_data() { return ""sv; }
template<> std::string_view checkpoint_data<cryptonote::network_type::MAINNET>();
template<> std::string_view checkpoint_data<cryptonote::network_type::DEVNET>();
template<> std::string_view checkpoint_data<cryptonote::network_type::TESTNET>();
namespace blocks {
inline std::string_view GetCheckpointsData(cryptonote::network_type network)
{
if (network == cryptonote::network_type::MAINNET) return checkpoint_data<cryptonote::network_type::MAINNET>();
if (network == cryptonote::network_type::TESTNET) return checkpoint_data<cryptonote::network_type::TESTNET>();
if (network == cryptonote::network_type::DEVNET) return checkpoint_data<cryptonote::network_type::DEVNET>();
template <cryptonote::network_type Network>
std::string_view checkpoint_data() {
return ""sv;
}
template <>
std::string_view checkpoint_data<cryptonote::network_type::MAINNET>();
template <>
std::string_view checkpoint_data<cryptonote::network_type::DEVNET>();
template <>
std::string_view checkpoint_data<cryptonote::network_type::TESTNET>();
inline std::string_view GetCheckpointsData(cryptonote::network_type network) {
if (network == cryptonote::network_type::MAINNET)
return checkpoint_data<cryptonote::network_type::MAINNET>();
if (network == cryptonote::network_type::TESTNET)
return checkpoint_data<cryptonote::network_type::TESTNET>();
if (network == cryptonote::network_type::DEVNET)
return checkpoint_data<cryptonote::network_type::DEVNET>();
return ""sv;
}
}
} // namespace blocks

196
src/checkpoints/checkpoints.cpp
View file

@ -31,36 +31,38 @@
#include "checkpoints.h"
#include "epee/string_tools.h"
#include "epee/storages/portable_storage_template_helper.h" // epee json include
#include "epee/serialization/keyvalue_serialization.h"
#include "cryptonote_core/service_node_rules.h"
#include <vector>
#include "blockchain_db/blockchain_db.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "common/oxen.h"
#include "blockchain_db/blockchain_db.h"
#include "common/file.h"
#include "common/fs-format.h"
#include "common/hex.h"
#include "common/oxen.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "cryptonote_core/service_node_rules.h"
#include "epee/serialization/keyvalue_serialization.h"
#include "epee/storages/portable_storage_template_helper.h" // epee json include
#include "epee/string_tools.h"
namespace cryptonote
{
namespace cryptonote {
static auto logcat = log::Cat("checkpoints");
bool checkpoint_t::check(crypto::hash const &hash) const
{
bool checkpoint_t::check(crypto::hash const& hash) const {
bool result = block_hash == hash;
if (result)
log::info(logcat, "CHECKPOINT PASSED FOR HEIGHT {} {}", height, block_hash);
else
log::warning(logcat, "CHECKPOINT FAILED FOR HEIGHT {}. EXPECTED HASH {}GIVEN HASH: {}", height, block_hash, hash);
log::warning(
logcat,
"CHECKPOINT FAILED FOR HEIGHT {}. EXPECTED HASH {}GIVEN HASH: {}",
height,
block_hash,
hash);
return result;
};
height_to_hash const HARDCODED_MAINNET_CHECKPOINTS[] =
{
height_to_hash const HARDCODED_MAINNET_CHECKPOINTS[] = {
{0, "08ff156d993012b0bdf2816c4bee47c9bbc7930593b70ee02574edddf15ee933"},
{1, "647997953a5ea9b5ab329c2291d4cbb08eed587c287e451eeeb2c79bab9b940f"},
{10, "4a7cd8b9bff380d48d6f3533a5e0509f8589cc77d18218b3f7218846e77738fc"},
@ -74,15 +76,13 @@ namespace cryptonote
{301187, "e23e4cf3a2fe3e9f0ffced5cc76426e5bdffd3aad822268f4ad63d82cb958559"},
};
crypto::hash get_newest_hardcoded_checkpoint(cryptonote::network_type nettype, uint64_t *height)
{
crypto::hash get_newest_hardcoded_checkpoint(cryptonote::network_type nettype, uint64_t* height) {
crypto::hash result{};
*height = 0;
if (nettype != network_type::MAINNET && nettype != network_type::TESTNET)
return result;
if (nettype == network_type::MAINNET)
{
if (nettype == network_type::MAINNET) {
uint64_t last_index = oxen::array_count(HARDCODED_MAINNET_CHECKPOINTS) - 1;
height_to_hash const& entry = HARDCODED_MAINNET_CHECKPOINTS[last_index];
@ -92,19 +92,16 @@ namespace cryptonote
return result;
}
bool load_checkpoints_from_json(const fs::path& json_hashfile_fullpath, std::vector<height_to_hash>& checkpoint_hashes)
{
if (std::error_code ec; !fs::exists(json_hashfile_fullpath, ec))
{
bool load_checkpoints_from_json(
const fs::path& json_hashfile_fullpath, std::vector<height_to_hash>& checkpoint_hashes) {
if (std::error_code ec; !fs::exists(json_hashfile_fullpath, ec)) {
log::debug(logcat, "Blockchain checkpoints file not found");
return true;
}
height_to_hash_json hashes;
if (std::string contents;
!tools::slurp_file(json_hashfile_fullpath, contents) ||
!epee::serialization::load_t_from_json(hashes, contents))
{
if (std::string contents; !tools::slurp_file(json_hashfile_fullpath, contents) ||
!epee::serialization::load_t_from_json(hashes, contents)) {
log::error(logcat, "Error loading checkpoints from {}", json_hashfile_fullpath);
return false;
}
@ -113,34 +110,35 @@ namespace cryptonote
return true;
}
bool checkpoints::get_checkpoint(uint64_t height, checkpoint_t &checkpoint) const
{
try
{
bool checkpoints::get_checkpoint(uint64_t height, checkpoint_t& checkpoint) const {
try {
auto guard = db_rtxn_guard(m_db);
return m_db->get_block_checkpoint(height, checkpoint);
}
catch (const std::exception &e)
{
log::error(logcat, "Get block checkpoint from DB failed at height: {}, what = {}", height, e.what());
} catch (const std::exception& e) {
log::error(
logcat,
"Get block checkpoint from DB failed at height: {}, what = {}",
height,
e.what());
return false;
}
}
//---------------------------------------------------------------------------
bool checkpoints::add_checkpoint(uint64_t height, const std::string& hash_str)
{
bool checkpoints::add_checkpoint(uint64_t height, const std::string& hash_str) {
crypto::hash h{};
bool r = tools::hex_to_type(hash_str, h);
CHECK_AND_ASSERT_MES(r, false, "Failed to parse checkpoint hash string into binary representation!");
CHECK_AND_ASSERT_MES(
r, false, "Failed to parse checkpoint hash string into binary representation!");
checkpoint_t checkpoint = {};
if (get_checkpoint(height, checkpoint))
{
if (get_checkpoint(height, checkpoint)) {
crypto::hash const& curr_hash = checkpoint.block_hash;
CHECK_AND_ASSERT_MES(h == curr_hash, false, "Checkpoint at given height already exists, and hash for new checkpoint was different!");
}
else
{
CHECK_AND_ASSERT_MES(
h == curr_hash,
false,
"Checkpoint at given height already exists, and hash for new checkpoint was "
"different!");
} else {
checkpoint.type = checkpoint_type::hardcoded;
checkpoint.height = height;
checkpoint.block_hash = h;
@ -149,19 +147,20 @@ namespace cryptonote
return r;
}
bool checkpoints::update_checkpoint(checkpoint_t const &checkpoint)
{
bool checkpoints::update_checkpoint(checkpoint_t const& checkpoint) {
// NOTE(oxen): Assumes checkpoint is valid
bool result = true;
bool batch_started = false;
try
{
try {
batch_started = m_db->batch_start();
m_db->update_block_checkpoint(checkpoint);
}
catch (const std::exception& e)
{
log::error(logcat, "Failed to add checkpoint with hash: {} at height: {}, what = {}", checkpoint.block_hash, checkpoint.height, e.what());
} catch (const std::exception& e) {
log::error(
logcat,
"Failed to add checkpoint with hash: {} at height: {}, what = {}",
checkpoint.block_hash,
checkpoint.height,
e.what());
result = false;
}
@ -170,10 +169,10 @@ namespace cryptonote
return result;
}
//---------------------------------------------------------------------------
void checkpoints::block_add(const block_add_info& info)
{
void checkpoints::block_add(const block_add_info& info) {
uint64_t const height = get_block_height(info.block);
if (height < service_nodes::CHECKPOINT_STORE_PERSISTENTLY_INTERVAL || info.block.major_version < hf::hf12_checkpointing)
if (height < service_nodes::CHECKPOINT_STORE_PERSISTENTLY_INTERVAL ||
info.block.major_version < hf::hf12_checkpointing)
return;
uint64_t end_cull_height = 0;
@ -182,27 +181,32 @@ namespace cryptonote
if (m_db->get_immutable_checkpoint(&immutable_checkpoint, height + 1))
end_cull_height = immutable_checkpoint.height;
}
uint64_t start_cull_height = (end_cull_height < service_nodes::CHECKPOINT_STORE_PERSISTENTLY_INTERVAL)
uint64_t start_cull_height =
(end_cull_height < service_nodes::CHECKPOINT_STORE_PERSISTENTLY_INTERVAL)
? 0
: end_cull_height - service_nodes::CHECKPOINT_STORE_PERSISTENTLY_INTERVAL;
if ((start_cull_height % service_nodes::CHECKPOINT_INTERVAL) > 0)
start_cull_height += (service_nodes::CHECKPOINT_INTERVAL - (start_cull_height % service_nodes::CHECKPOINT_INTERVAL));
start_cull_height +=
(service_nodes::CHECKPOINT_INTERVAL -
(start_cull_height % service_nodes::CHECKPOINT_INTERVAL));
m_last_cull_height = std::max(m_last_cull_height, start_cull_height);
auto guard = db_wtxn_guard(m_db);
for (; m_last_cull_height < end_cull_height; m_last_cull_height += service_nodes::CHECKPOINT_INTERVAL)
{
for (; m_last_cull_height < end_cull_height;
m_last_cull_height += service_nodes::CHECKPOINT_INTERVAL) {
if (m_last_cull_height % service_nodes::CHECKPOINT_STORE_PERSISTENTLY_INTERVAL == 0)
continue;
try
{
try {
m_db->remove_block_checkpoint(m_last_cull_height);
}
catch (const std::exception &e)
{
log::error(logcat, "Pruning block checkpoint on block added failed non-trivially at height: {}, what = {}", m_last_cull_height, e.what());
} catch (const std::exception& e) {
log::error(
logcat,
"Pruning block checkpoint on block added failed non-trivially at height: {}, "
"what = {}",
m_last_cull_height,
e.what());
}
}
@ -210,33 +214,31 @@ namespace cryptonote
update_checkpoint(*info.checkpoint);
}
//---------------------------------------------------------------------------
void checkpoints::blockchain_detached(uint64_t height)
{
void checkpoints::blockchain_detached(uint64_t height) {
m_last_cull_height = std::min(m_last_cull_height, height);
checkpoint_t top_checkpoint;
auto guard = db_wtxn_guard(m_db);
if (m_db->get_top_checkpoint(top_checkpoint))
{
if (m_db->get_top_checkpoint(top_checkpoint)) {
uint64_t start_height = top_checkpoint.height;
for (size_t delete_height = start_height;
delete_height >= height && delete_height >= service_nodes::CHECKPOINT_INTERVAL;
delete_height -= service_nodes::CHECKPOINT_INTERVAL)
{
try
{
delete_height -= service_nodes::CHECKPOINT_INTERVAL) {
try {
m_db->remove_block_checkpoint(delete_height);
}
catch (const std::exception &e)
{
log::error(logcat, "Remove block checkpoint on detach failed non-trivially at height: {}, what = {}", delete_height, e.what());
} catch (const std::exception& e) {
log::error(
logcat,
"Remove block checkpoint on detach failed non-trivially at height: {}, "
"what = {}",
delete_height,
e.what());
}
}
}
}
//---------------------------------------------------------------------------
bool checkpoints::is_in_checkpoint_zone(uint64_t height) const
{
bool checkpoints::is_in_checkpoint_zone(uint64_t height) const {
uint64_t top_checkpoint_height = 0;
checkpoint_t top_checkpoint;
if (m_db->get_top_checkpoint(top_checkpoint))
@ -245,12 +247,17 @@ namespace cryptonote
return height <= top_checkpoint_height;
}
//---------------------------------------------------------------------------
bool checkpoints::check_block(uint64_t height, const crypto::hash& h, bool* is_a_checkpoint, bool *service_node_checkpoint) const
{
bool checkpoints::check_block(
uint64_t height,
const crypto::hash& h,
bool* is_a_checkpoint,
bool* service_node_checkpoint) const {
checkpoint_t checkpoint;
bool found = get_checkpoint(height, checkpoint);
if (is_a_checkpoint) *is_a_checkpoint = found;
if (service_node_checkpoint) *service_node_checkpoint = false;
if (is_a_checkpoint)
*is_a_checkpoint = found;
if (service_node_checkpoint)
*service_node_checkpoint = false;
if (!found)
return true;
@ -262,8 +269,8 @@ namespace cryptonote
return result;
}
//---------------------------------------------------------------------------
bool checkpoints::is_alternative_block_allowed(uint64_t blockchain_height, uint64_t block_height, bool *service_node_checkpoint)
{
bool checkpoints::is_alternative_block_allowed(
uint64_t blockchain_height, uint64_t block_height, bool* service_node_checkpoint) {
if (service_node_checkpoint)
*service_node_checkpoint = false;
@ -271,15 +278,15 @@ namespace cryptonote
return false;
{
std::vector<checkpoint_t> const first_checkpoint = m_db->get_checkpoints_range(0, blockchain_height, 1);
std::vector<checkpoint_t> const first_checkpoint =
m_db->get_checkpoints_range(0, blockchain_height, 1);
if (first_checkpoint.empty() || blockchain_height < first_checkpoint[0].height)
return true;
}
checkpoint_t immutable_checkpoint;
uint64_t immutable_height = 0;
if (m_db->get_immutable_checkpoint(&immutable_checkpoint, blockchain_height))
{
if (m_db->get_immutable_checkpoint(&immutable_checkpoint, blockchain_height)) {
immutable_height = immutable_checkpoint.height;
if (service_node_checkpoint)
*service_node_checkpoint = (immutable_checkpoint.type == checkpoint_type::service_node);
@ -290,8 +297,7 @@ namespace cryptonote
return result;
}
//---------------------------------------------------------------------------
uint64_t checkpoints::get_max_height() const
{
uint64_t checkpoints::get_max_height() const {
uint64_t result = 0;
checkpoint_t top_checkpoint;
if (m_db->get_top_checkpoint(top_checkpoint))
@ -300,8 +306,7 @@ namespace cryptonote
return result;
}
//---------------------------------------------------------------------------
bool checkpoints::init(network_type nettype, BlockchainDB *db)
{
bool checkpoints::init(network_type nettype, BlockchainDB* db) {
*this = {};
m_db = db;
m_nettype = nettype;
@ -309,10 +314,8 @@ namespace cryptonote
if (db->is_read_only())
return true;
if (nettype == network_type::MAINNET)
{
for (size_t i = 0; i < oxen::array_count(HARDCODED_MAINNET_CHECKPOINTS); ++i)
{
if (nettype == network_type::MAINNET) {
for (size_t i = 0; i < oxen::array_count(HARDCODED_MAINNET_CHECKPOINTS); ++i) {
height_to_hash const& checkpoint = HARDCODED_MAINNET_CHECKPOINTS[i];
bool added = add_checkpoint(checkpoint.height, checkpoint.hash);
CHECK_AND_ASSERT(added, false);
@ -322,5 +325,4 @@ namespace cryptonote
return true;
}
}
} // namespace cryptonote

48
src/checkpoints/checkpoints.h
View file

@ -31,38 +31,33 @@
#pragma once
#include <vector>
#include "common/fs.h"
#include "crypto/hash.h"
#include "cryptonote_basic/cryptonote_basic_impl.h"
#include "cryptonote_config.h"
#include "cryptonote_core/service_node_voting.h"
#include "cryptonote_basic/cryptonote_basic_impl.h"
#include "common/fs.h"
namespace cryptonote
{
namespace cryptonote {
constexpr std::string_view JSON_HASH_FILE_NAME = "checkpoints.json"sv;
enum struct checkpoint_type
{
enum struct checkpoint_type {
hardcoded,
service_node,
count,
};
struct checkpoint_t
{
struct checkpoint_t {
uint8_t version = 0;
checkpoint_type type;
uint64_t height;
crypto::hash block_hash;
std::vector<service_nodes::quorum_signature> signatures; // Only service node checkpoints use signatures
std::vector<service_nodes::quorum_signature>
signatures; // Only service node checkpoints use signatures
uint64_t prev_height; // TODO(doyle): Unused
bool check(crypto::hash const& block_hash) const;
static char const *type_to_string(checkpoint_type type)
{
switch(type)
{
static char const* type_to_string(checkpoint_type type) {
switch (type) {
case checkpoint_type::hardcoded: return "Hardcoded";
case checkpoint_type::service_node: return "ServiceNode";
default: assert(false); return "XXUnhandledVersion";
@ -79,8 +74,7 @@ namespace cryptonote
END_SERIALIZE()
};
struct height_to_hash
{
struct height_to_hash {
uint64_t height; //!< the height of the checkpoint
std::string hash; //!< the hash for the checkpoint
BEGIN_KV_SERIALIZE_MAP()
@ -100,7 +94,8 @@ namespace cryptonote
};
crypto::hash get_newest_hardcoded_checkpoint(cryptonote::network_type nettype, uint64_t* height);
bool load_checkpoints_from_json (const fs::path& json_hashfile_fullpath, std::vector<height_to_hash>& checkpoint_hashes);
bool load_checkpoints_from_json(
const fs::path& json_hashfile_fullpath, std::vector<height_to_hash>& checkpoint_hashes);
/**
* @brief A container for blockchain checkpoints
@ -109,8 +104,7 @@ namespace cryptonote
* Some of these are compiled-in, while others can be loaded at runtime
* either from a json file or via DNS from a checkpoint-hosting server.
*/
class checkpoints
{
class checkpoints {
public:
void block_add(const block_add_info& info);
void blockchain_detached(uint64_t height);
@ -153,13 +147,18 @@ namespace cryptonote
* @param height the height to be checked
* @param h the hash to be checked
* @param blockchain the blockchain to query ancestor blocks from the current height
* @param is_a_checkpoint optional return-by-pointer if there is a checkpoint at the given height
* @param is_a_checkpoint optional return-by-pointer if there is a checkpoint at the given
* height
*
* @return true if there is no checkpoint at the given height,
* true if the passed parameters match the stored checkpoint,
* false otherwise
*/
bool check_block(uint64_t height, const crypto::hash& h, bool *is_a_checkpoint = nullptr, bool *service_node_checkpoint = nullptr) const;
bool check_block(
uint64_t height,
const crypto::hash& h,
bool* is_a_checkpoint = nullptr,
bool* service_node_checkpoint = nullptr) const;
/**
* @brief checks if alternate chain blocks should be kept for a given height and updates
@ -176,7 +175,10 @@ namespace cryptonote
* @return true if alternate blocks are allowed given the parameters,
* otherwise false
*/
bool is_alternative_block_allowed(uint64_t blockchain_height, uint64_t block_height, bool *service_node_checkpoint = nullptr);
bool is_alternative_block_allowed(
uint64_t blockchain_height,
uint64_t block_height,
bool* service_node_checkpoint = nullptr);
/**
* @brief gets the highest checkpoint height
@ -200,4 +202,4 @@ namespace cryptonote
BlockchainDB* m_db;
};
}
} // namespace cryptonote

33
src/common/aligned.c
View file

@ -26,20 +26,22 @@
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdint.h>
#include <string.h>
#include "aligned.h"
static inline int is_power_of_2(size_t n) { return n && (n & (n-1)) == 0; }
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
static inline int is_power_of_2(size_t n) {
return n && (n & (n - 1)) == 0;
}
#define MAGIC 0xaa0817161500ff81
#define MAGIC_FREED 0xaa0817161500ff82
static void local_abort(const char *msg)
{
static void local_abort(const char* msg) {
fprintf(stderr, "%s\n", msg);
#ifdef NDEBUG
_exit(1);
@ -48,16 +50,14 @@ static void local_abort(const char *msg)
#endif
}
typedef struct
{
typedef struct {
uint64_t magic;
void* raw;
size_t bytes;
size_t align;
} control;
void *aligned_malloc(size_t bytes, size_t align)
{
void* aligned_malloc(size_t bytes, size_t align) {
void *raw, *ptr;
control* ctrl;
@ -80,15 +80,13 @@ void *aligned_malloc(size_t bytes, size_t align)
return ptr;
}
void *aligned_realloc(void *ptr, size_t bytes, size_t align)
{
void* aligned_realloc(void* ptr, size_t bytes, size_t align) {
void *raw, *ptr2;
control *ctrl, *ctrl2;
if (!ptr)
return aligned_malloc(bytes, align);
if (!bytes)
{
if (!bytes) {
aligned_free(ptr);
return NULL;
}
@ -125,8 +123,7 @@ void *aligned_realloc(void *ptr, size_t bytes, size_t align)
return ptr2;
}
void aligned_free(void *ptr)
{
void aligned_free(void* ptr) {
if (!ptr)
return;
control* ctrl = ((control*)ptr) - 1;

26
src/common/apply_permutation.h
View file

@ -32,30 +32,26 @@
#pragma once
#include <vector>
#include <functional>
#include <vector>
#include "logging/oxen_logger.h"
namespace tools
{
namespace tools {
template <typename F>
void apply_permutation(std::vector<size_t> permutation, const F &swap)
{
void apply_permutation(std::vector<size_t> permutation, const F& swap) {
// sanity check
for (size_t n = 0; n < permutation.size(); ++n)
if (std::find(permutation.begin(), permutation.end(), n) == permutation.end())
{
if (std::find(permutation.begin(), permutation.end(), n) == permutation.end()) {
log::error(globallogcat, "Bad permutation");
throw std::runtime_error("Bad permutation");
return;
}
for (size_t i = 0; i < permutation.size(); ++i)
{
for (size_t i = 0; i < permutation.size(); ++i) {
size_t current = i;
while (i != permutation[current])
{
while (i != permutation[current]) {
size_t next = permutation[current];
swap(current, next);
permutation[current] = current;
@ -66,10 +62,8 @@ void apply_permutation(std::vector<size_t> permutation, const F &swap)
}
template <typename T>
void apply_permutation(const std::vector<size_t> &permutation, std::vector<T> &v)
{
if (permutation.size() != v.size())
{
void apply_permutation(const std::vector<size_t>& permutation, std::vector<T>& v) {
if (permutation.size() != v.size()) {
log::error(globallogcat, "Mismatched vector sizes");
throw std::runtime_error("Mismatched vector sizes");
return;
@ -77,4 +71,4 @@ void apply_permutation(const std::vector<size_t> &permutation, std::vector<T> &v
apply_permutation(permutation, [&v](size_t i0, size_t i1) { std::swap(v[i0], v[i1]); });
}
}
} // namespace tools

117
src/common/base58.cpp
View file

@ -32,46 +32,42 @@
#include <cassert>
#include <cstring>
#include <vector>
#include <string_view>
#include <vector>
#include "crypto/hash.h"
#include "epee/int-util.h"
#include "varint.h"
namespace tools
{
namespace tools {
using namespace std::literals;
namespace base58
{
namespace
{
constexpr std::string_view alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"sv;
namespace base58 {
namespace {
constexpr std::string_view alphabet =
"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"sv;
constexpr size_t full_block_size = 8;
constexpr std::array<uint8_t, full_block_size + 1> encoded_block_sizes = {0, 2, 3, 5, 6, 7, 9, 10, 11};
constexpr std::array<uint8_t, full_block_size + 1> encoded_block_sizes = {
0, 2, 3, 5, 6, 7, 9, 10, 11};
constexpr size_t full_encoded_block_size = encoded_block_sizes.back();
constexpr std::array<int8_t, full_encoded_block_size + 1> decoded_block_sizes = {0, -1, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8};
constexpr std::array<int8_t, full_encoded_block_size + 1> decoded_block_sizes = {
0, -1, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8};
constexpr size_t addr_checksum_size = 4;
struct reverse_alphabet_table
{
struct reverse_alphabet_table {
std::array<int8_t, 256> from_b58_lut;
constexpr reverse_alphabet_table() noexcept : from_b58_lut{}
{
constexpr reverse_alphabet_table() noexcept : from_b58_lut{} {
for (size_t i = 0; i < from_b58_lut.size(); ++i)
from_b58_lut[i] = -1;
for (size_t i = 0; i < alphabet.size(); i++)
from_b58_lut[alphabet[i]] = i;
}
constexpr int8_t operator[](char letter) const
{
constexpr int8_t operator[](char letter) const {
return from_b58_lut[static_cast<unsigned char>(letter)];
}
} constexpr reverse_alphabet;
uint64_t uint_8be_to_64(const uint8_t* data, size_t size)
{
uint64_t uint_8be_to_64(const uint8_t* data, size_t size) {
assert(1 <= size && size <= sizeof(uint64_t));
uint64_t res = 0;
@ -79,22 +75,19 @@ namespace tools
return SWAP64BE(res);
}
void uint_64_to_8be(uint64_t num, size_t size, uint8_t* data)
{
void uint_64_to_8be(uint64_t num, size_t size, uint8_t* data) {
assert(1 <= size && size <= sizeof(uint64_t));
uint64_t num_be = SWAP64BE(num);
memcpy(data, reinterpret_cast<uint8_t*>(&num_be) + sizeof(uint64_t) - size, size);
}
void encode_block(const char* block, size_t size, char* res)
{
void encode_block(const char* block, size_t size, char* res) {
assert(1 <= size && size <= full_block_size);
uint64_t num = uint_8be_to_64(reinterpret_cast<const uint8_t*>(block), size);
int i = static_cast<int>(encoded_block_sizes[size]) - 1;
while (0 < num)
{
while (0 < num) {
uint64_t remainder = num % alphabet.size();
num /= alphabet.size();
res[i] = alphabet[remainder];
@ -102,8 +95,7 @@ namespace tools
}
}
bool decode_block(const char* block, size_t size, char* res)
{
bool decode_block(const char* block, size_t size, char* res) {
assert(1 <= size && size <= full_encoded_block_size);
int res_size = decoded_block_sizes[size];
@ -112,8 +104,7 @@ namespace tools
uint64_t res_num = 0;
uint64_t order = 1;
for (size_t i = size - 1; i < size; --i)
{
for (size_t i = size - 1; i < size; --i) {
auto digit = reverse_alphabet[block[i]];
if (digit < 0)
return false; // Invalid symbol
@ -127,42 +118,45 @@ namespace tools
order *= alphabet.size(); // Never overflows, 58^10 < 2^64
}
if (static_cast<size_t>(res_size) < full_block_size && (UINT64_C(1) << (8 * res_size)) <= res_num)
if (static_cast<size_t>(res_size) < full_block_size &&
(UINT64_C(1) << (8 * res_size)) <= res_num)
return false; // Overflow
uint_64_to_8be(res_num, res_size, reinterpret_cast<uint8_t*>(res));
return true;
}
}
} // namespace
std::string encode(std::string_view data)
{
std::string encode(std::string_view data) {
if (data.empty())
return std::string();
size_t full_block_count = data.size() / full_block_size;
size_t last_block_size = data.size() % full_block_size;
size_t res_size = full_block_count * full_encoded_block_size + encoded_block_sizes[last_block_size];
size_t res_size =
full_block_count * full_encoded_block_size + encoded_block_sizes[last_block_size];
std::string res(res_size, alphabet[0]);
for (size_t i = 0; i < full_block_count; ++i)
{
encode_block(data.data() + i * full_block_size, full_block_size, &res[i * full_encoded_block_size]);
for (size_t i = 0; i < full_block_count; ++i) {
encode_block(
data.data() + i * full_block_size,
full_block_size,
&res[i * full_encoded_block_size]);
}
if (0 < last_block_size)
{
encode_block(data.data() + full_block_count * full_block_size, last_block_size, &res[full_block_count * full_encoded_block_size]);
if (0 < last_block_size) {
encode_block(
data.data() + full_block_count * full_block_size,
last_block_size,
&res[full_block_count * full_encoded_block_size]);
}
return res;
}
bool decode(std::string_view enc, std::string& data)
{
if (enc.empty())
{
bool decode(std::string_view enc, std::string& data) {
if (enc.empty()) {
data.clear();
return true;
}
@ -175,15 +169,18 @@ namespace tools
size_t data_size = full_block_count * full_block_size + last_block_decoded_size;
data.resize(data_size, 0);
for (size_t i = 0; i < full_block_count; ++i)
{
if (!decode_block(enc.data() + i * full_encoded_block_size, full_encoded_block_size, &data[i * full_block_size]))
for (size_t i = 0; i < full_block_count; ++i) {
if (!decode_block(
enc.data() + i * full_encoded_block_size,
full_encoded_block_size,
&data[i * full_block_size]))
return false;
}
if (0 < last_block_size)
{
if (!decode_block(enc.data() + full_block_count * full_encoded_block_size, last_block_size,
if (0 < last_block_size) {
if (!decode_block(
enc.data() + full_block_count * full_encoded_block_size,
last_block_size,
&data[full_block_count * full_block_size]))
return false;
}
@ -191,8 +188,7 @@ namespace tools
return true;
}
std::string encode_addr(uint64_t tag, std::string_view data)
{
std::string encode_addr(uint64_t tag, std::string_view data) {
std::string buf = get_varint_data(tag);
buf += data;
crypto::hash hash = crypto::cn_fast_hash(buf.data(), buf.size());
@ -201,12 +197,13 @@ namespace tools
return encode(buf);
}
bool decode_addr(std::string_view addr, uint64_t& tag, std::string& data)
{
bool decode_addr(std::string_view addr, uint64_t& tag, std::string& data) {
std::string addr_data;
bool r = decode(addr, addr_data);
if (!r) return false;
if (addr_data.size() <= addr_checksum_size) return false;
if (!r)
return false;
if (addr_data.size() <= addr_checksum_size)
return false;
std::string checksum(addr_checksum_size, '\0');
checksum = addr_data.substr(addr_data.size() - addr_checksum_size);
@ -214,13 +211,15 @@ namespace tools
addr_data.resize(addr_data.size() - addr_checksum_size);
crypto::hash hash = crypto::cn_fast_hash(addr_data.data(), addr_data.size());
std::string expected_checksum(reinterpret_cast<const char*>(&hash), addr_checksum_size);
if (expected_checksum != checksum) return false;
if (expected_checksum != checksum)
return false;
int read = tools::read_varint(addr_data.begin(), addr_data.end(), tag);
if (read <= 0) return false;
if (read <= 0)
return false;
data = addr_data.substr(read);
return true;
}
}
}
} // namespace base58
} // namespace tools

8
src/common/base58.h
View file

@ -34,14 +34,10 @@
#include <string>
#include <string_view>
namespace tools
{
namespace base58
{
namespace tools { namespace base58 {
std::string encode(std::string_view data);
bool decode(std::string_view enc, std::string& data);
std::string encode_addr(uint64_t tag, std::string_view data);
bool decode_addr(std::string_view addr, uint64_t& tag, std::string& data);
}
}
}} // namespace tools::base58

24
src/common/boost_serialization_helper.h
View file

@ -31,18 +31,16 @@
#pragma once
#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/portable_binary_oarchive.hpp>
#include <boost/archive/portable_binary_iarchive.hpp>
#include "epee/misc_log_ex.h"
#include "fs.h"
#include <boost/archive/portable_binary_oarchive.hpp>
#include <fstream>
#include "epee/misc_log_ex.h"
#include "fs.h"
namespace tools
{
namespace tools {
template <class T>
bool serialize_obj_to_file(T& obj, const fs::path& file_path)
{
bool serialize_obj_to_file(T& obj, const fs::path& file_path) {
TRY_ENTRY();
fs::ofstream data_file{file_path, std::ios::binary | std::ios::trunc};
if (data_file.fail())
@ -60,20 +58,16 @@ namespace tools
}
template <class T>
bool unserialize_obj_from_file(T& obj, const fs::path& file_path)
{
bool unserialize_obj_from_file(T& obj, const fs::path& file_path) {
TRY_ENTRY();
fs::ifstream data_file{file_path, std::ios_base::binary};
if (data_file.fail())
return false;
try
{
try {
// first try reading in portable mode
boost::archive::portable_binary_iarchive{data_file} >> obj;
}
catch(...)
{
} catch (...) {
// if failed, try reading in unportable mode
auto unportable = file_path;
unportable += ".unportable";
@ -87,4 +81,4 @@ namespace tools
return !data_file.fail();
CATCH_ENTRY_L0("unserialize_obj_from_file", false);
}
}
} // namespace tools

5
src/common/combinator.cpp
View file

@ -32,8 +32,7 @@
namespace tools {
uint64_t combinations_count(uint32_t k, uint32_t n)
{
uint64_t combinations_count(uint32_t k, uint32_t n) {
if (k > n) {
throw std::runtime_error("k must not be greater than n");
}
@ -47,4 +46,4 @@ uint64_t combinations_count(uint32_t k, uint32_t n)
return c;
}
}
} // namespace tools

19
src/common/combinator.h
View file

@ -30,10 +30,10 @@
#pragma once
#include <vector>
#include <stdexcept>
#include <cstddef>
#include <cstdint>
#include <stdexcept>
#include <vector>
namespace tools {
@ -55,15 +55,12 @@ private:
};
template <typename T>
std::vector<std::vector<T>> Combinator<T>::combine(size_t k)
{
if (k > origin.size())
{
std::vector<std::vector<T>> Combinator<T>::combine(size_t k) {
if (k > origin.size()) {
throw std::runtime_error("k must be smaller than elements number");
}
if (k == 0)
{
if (k == 0) {
throw std::runtime_error("k must be greater than zero");
}
@ -73,12 +70,10 @@ std::vector<std::vector<T>> Combinator<T>::combine(size_t k)
}
template <typename T>
void Combinator<T>::doCombine(size_t from, size_t k)
{
void Combinator<T>::doCombine(size_t from, size_t k) {
current.push_back(0);
for (size_t i = from; i <= origin.size() - k; ++i)
{
for (size_t i = from; i <= origin.size() - k; ++i) {
current.back() = i;
if (k > 1) {

39
src/common/command_line.cpp
View file

@ -29,6 +29,7 @@
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
#include "command_line.h"
#include "common/i18n.h"
#include "common/string_util.h"
#ifdef HAVE_READLINE
@ -39,8 +40,7 @@
#include "windows.h"
#endif
namespace command_line
{
namespace command_line {
const arg_descriptor<bool> arg_help = {"help", "Produce help message"};
const arg_descriptor<bool> arg_version = {"version", "Output version information"};
@ -51,8 +51,8 @@ const arg_descriptor<bool> arg_version = {"version", "Output version information
#ifdef __linux__
extern "C" {
#include <sys/ioctl.h>
#include <stdio.h>
#include <sys/ioctl.h>
#include <unistd.h>
}
std::pair<unsigned, unsigned> terminal_size() {
@ -64,11 +64,12 @@ std::pair<unsigned, unsigned> terminal_size() {
#else
std::pair<unsigned, unsigned> terminal_size() { return {0, 0}; }
std::pair<unsigned, unsigned> terminal_size() {
return {0, 0};
}
#endif
std::pair<unsigned, unsigned> boost_option_sizes() {
std::pair<unsigned, unsigned> result;
@ -76,13 +77,13 @@ std::pair<unsigned, unsigned> boost_option_sizes() {
terminal_size().first,
boost::program_options::options_description::m_default_line_length);
result.second = result.first - boost::program_options::options_description::m_default_line_length / 2;
result.second =
result.first - boost::program_options::options_description::m_default_line_length / 2;
return result;
}
void clear_screen()
{
void clear_screen() {
#ifdef HAVE_READLINE
rdln::clear_screen();
#else
@ -91,13 +92,13 @@ void clear_screen()
std::cout << "\033[2J"; // clear current screen only, scrollback is still around
std::cout << "\033[3J"; // does nothing, should clear current screen and scrollback
std::cout << "\033[1;1H"; // move cursor top/left
std::cout << "\r \r" << std::flush; // erase odd chars if the ANSI codes were printed raw
std::cout << "\r \r"
<< std::flush; // erase odd chars if the ANSI codes were printed raw
#ifdef _WIN32
COORD coord{0, 0};
CONSOLE_SCREEN_BUFFER_INFO csbi;
HANDLE h = GetStdHandle(STD_OUTPUT_HANDLE);
if (GetConsoleScreenBufferInfo(h, &csbi))
{
if (GetConsoleScreenBufferInfo(h, &csbi)) {
DWORD cbConSize = csbi.dwSize.X * csbi.dwSize.Y, w;
FillConsoleOutputCharacter(h, (TCHAR)' ', cbConSize, coord, &w);
if (GetConsoleScreenBufferInfo(h, &csbi))
@ -108,23 +109,19 @@ void clear_screen()
#endif
}
bool handle_error_helper(const boost::program_options::options_description& desc, std::function<bool()> parser) {
try
{
bool handle_error_helper(
const boost::program_options::options_description& desc, std::function<bool()> parser) {
try {
return parser();
}
catch (const std::exception& e)
{
} catch (const std::exception& e) {
std::cerr << "Failed to parse arguments: " << e.what() << std::endl;
std::cerr << desc << std::endl;
return false;
}
catch (...)
{
} catch (...) {
std::cerr << "Failed to parse arguments: unknown exception" << std::endl;
std::cerr << desc << std::endl;
return false;
}
}
}
} // namespace command_line

154
src/common/command_line.h
View file

@ -30,47 +30,55 @@
#pragma once
#include <functional>
#include <array>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/variables_map.hpp>
#include <functional>
#include <type_traits>
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/variables_map.hpp>
#include "common/string_util.h"
#include "common/i18n.h"
#include "common/format.h"
#include "common/i18n.h"
#include "common/string_util.h"
#include "logging/oxen_logger.h"
namespace command_line
{
namespace command_line {
namespace log = oxen::log;
inline const char* tr(const char* str) { return i18n_translate(str, "command_line"); }
inline const char* tr(const char* str) {
return i18n_translate(str, "command_line");
}
/// @return True if `str` is (case-insensitively) y, yes, a potentially translated yes, or any of
/// the optional extra arguments passed in.
template <typename S, typename... More>
bool is_yes(const S& str, const More&... more) { return tools::string_iequal_any(str, "y", "yes", tr("yes"), more...); }
bool is_yes(const S& str, const More&... more) {
return tools::string_iequal_any(str, "y", "yes", tr("yes"), more...);
}
/// @return True if `str` is (case-insensitively) n, no, or a potentially translated no, or any of
/// the optional extra arguments passed in.
template <typename S, typename... More>
bool is_no(const S& str, const More&... more) { return tools::string_iequal_any(str, "n", "no", tr("no"), more...); }
bool is_no(const S& str, const More&... more) {
return tools::string_iequal_any(str, "n", "no", tr("no"), more...);
}
/// @return True if `str` is (case-insensitively) c, cancel, or a potentially translated cancel,
/// or any of the optional extra arguments passed in.
template <typename S, typename... More>
bool is_cancel(const S& str, const More&... more) { return tools::string_iequal_any(str, "c", "cancel", tr("cancel"), more...); }
bool is_cancel(const S& str, const More&... more) {
return tools::string_iequal_any(str, "c", "cancel", tr("cancel"), more...);
}
/// @return True if `str` is (case-insensitively) b, back, or a potentially translated back, or
/// any of the optional extra arguments passed in.
template <typename S, typename... More>
bool is_back(const S& str, const More&... more) { return tools::string_iequal_any(str, "b", "back", tr("back"), more...); }
bool is_back(const S& str, const More&... more) {
return tools::string_iequal_any(str, "b", "back", tr("back"), more...);
}
template <typename T, bool required = false, bool dependent = false, int NUM_DEPS = 1>
struct arg_descriptor;
template <typename T>
struct arg_descriptor<T, false>
{
struct arg_descriptor<T, false> {
using value_type = T;
const char* name;
@ -80,8 +88,7 @@ namespace command_line
};
template <typename T>
struct arg_descriptor<T, true>
{
struct arg_descriptor<T, true> {
static_assert(!std::is_same_v<T, bool>, "Boolean switch can't be required");
using value_type = T;
@ -91,8 +98,7 @@ namespace command_line
};
template <typename T>
struct arg_descriptor<T, false, true>
{
struct arg_descriptor<T, false, true> {
using value_type = T;
const char* name;
@ -107,8 +113,7 @@ namespace command_line
};
template <typename T, int NUM_DEPS>
struct arg_descriptor<T, false, true, NUM_DEPS>
{
struct arg_descriptor<T, false, true, NUM_DEPS> {
using value_type = T;
const char* name;
@ -123,14 +128,13 @@ namespace command_line
};
template <typename T>
boost::program_options::typed_value<T, char>* make_semantic(const arg_descriptor<T, true>& /*arg*/)
{
boost::program_options::typed_value<T, char>* make_semantic(
const arg_descriptor<T, true>& /*arg*/) {
return boost::program_options::value<T>()->required();
}
template <typename T>
boost::program_options::typed_value<T, char>* make_semantic(const arg_descriptor<T, false>& arg)
{
boost::program_options::typed_value<T, char>* make_semantic(const arg_descriptor<T, false>& arg) {
auto semantic = boost::program_options::value<T>();
if (!arg.not_use_default)
semantic->default_value(arg.default_value);
@ -146,35 +150,36 @@ namespace command_line
std::string arg_stringify(const std::vector<T>& v) {
return "{{{}}}"_format(fmt::join(v, ","));
}
}
} // namespace
template <typename T>
boost::program_options::typed_value<T, char>* make_semantic(const arg_descriptor<T, false, true>& arg)
{
boost::program_options::typed_value<T, char>* make_semantic(
const arg_descriptor<T, false, true>& arg) {
auto semantic = boost::program_options::value<T>();
if (!arg.not_use_default) {
auto default_display = "{}, {} if '{}'"_format(
arg_stringify(arg.depf(false, true, arg.default_value)),
arg_stringify(arg.depf(true, true, arg.default_value)),
arg.ref.name);
semantic->default_value(arg.depf(arg.ref.default_value, true, arg.default_value), default_display);
semantic->default_value(
arg.depf(arg.ref.default_value, true, arg.default_value), default_display);
}
return semantic;
}
template <typename T, int NUM_DEPS>
boost::program_options::typed_value<T, char>* make_semantic(const arg_descriptor<T, false, true, NUM_DEPS>& arg)
{
boost::program_options::typed_value<T, char>* make_semantic(
const arg_descriptor<T, false, true, NUM_DEPS>& arg) {
auto semantic = boost::program_options::value<T>();
if (!arg.not_use_default) {
std::array<bool, NUM_DEPS> depval;
depval.fill(false);
auto default_display = arg_stringify(arg.depf(depval, true, arg.default_value));
for (size_t i = 0; i < depval.size(); ++i)
{
for (size_t i = 0; i < depval.size(); ++i) {
depval.fill(false);
depval[i] = true;
fmt::format_to(std::back_inserter(default_display),
fmt::format_to(
std::back_inserter(default_display),
", {} if '{}'",
arg_stringify(arg.depf(depval, true, arg.default_value)),
arg.ref[i]->name);
@ -187,8 +192,8 @@ namespace command_line
}
template <typename T>
boost::program_options::typed_value<T, char>* make_semantic(const arg_descriptor<T, false>& arg, const T& def)
{
boost::program_options::typed_value<T, char>* make_semantic(
const arg_descriptor<T, false>& arg, const T& def) {
auto semantic = boost::program_options::value<T>();
if (!arg.not_use_default)
semantic->default_value(def);
@ -196,18 +201,19 @@ namespace command_line
}
template <typename T>
boost::program_options::typed_value<std::vector<T>, char>* make_semantic(const arg_descriptor<std::vector<T>, false>& /*arg*/)
{
boost::program_options::typed_value<std::vector<T>, char>* make_semantic(
const arg_descriptor<std::vector<T>, false>& /*arg*/) {
auto semantic = boost::program_options::value<std::vector<T>>();
semantic->default_value(std::vector<T>(), "");
return semantic;
}
template <typename T, bool required, bool dependent, int NUM_DEPS>
void add_arg(boost::program_options::options_description& description, const arg_descriptor<T, required, dependent, NUM_DEPS>& arg, bool unique = true)
{
if (0 != description.find_nothrow(arg.name, false))
{
void add_arg(
boost::program_options::options_description& description,
const arg_descriptor<T, required, dependent, NUM_DEPS>& arg,
bool unique = true) {
if (0 != description.find_nothrow(arg.name, false)) {
if (!unique)
log::error(globallogcat, "Argument already exists: {}", arg.name);
return;
@ -217,10 +223,12 @@ namespace command_line
}
template <typename T>
void add_arg(boost::program_options::options_description& description, const arg_descriptor<T, false>& arg, const T& def, bool unique = true)
{
if (0 != description.find_nothrow(arg.name, false))
{
void add_arg(
boost::program_options::options_description& description,
const arg_descriptor<T, false>& arg,
const T& def,
bool unique = true) {
if (0 != description.find_nothrow(arg.name, false)) {
if (!unique)
log::error(globallogcat, "Argument already exists: {}", arg.name);
return;
@ -230,10 +238,11 @@ namespace command_line
}
template <>
inline void add_arg(boost::program_options::options_description& description, const arg_descriptor<bool, false>& arg, bool unique)
{
if (0 != description.find_nothrow(arg.name, false))
{
inline void add_arg(
boost::program_options::options_description& description,
const arg_descriptor<bool, false>& arg,
bool unique) {
if (0 != description.find_nothrow(arg.name, false)) {
if (!unique)
log::error(globallogcat, "Argument already exists: {}", arg.name);
return;
@ -243,42 +252,48 @@ namespace command_line
}
template <typename charT>
boost::program_options::basic_parsed_options<charT> parse_command_line(int argc, const charT* const argv[],
const boost::program_options::options_description& desc, bool allow_unregistered = false)
{
boost::program_options::basic_parsed_options<charT> parse_command_line(
int argc,
const charT* const argv[],
const boost::program_options::options_description& desc,
bool allow_unregistered = false) {
auto parser = boost::program_options::command_line_parser(argc, argv);
parser.options(desc);
if (allow_unregistered)
{
if (allow_unregistered) {
parser.allow_unregistered();
}
return parser.run();
}
bool handle_error_helper(const boost::program_options::options_description& desc, std::function<bool()> parser);
bool handle_error_helper(
const boost::program_options::options_description& desc, std::function<bool()> parser);
template <typename T, bool required, bool dependent, int NUM_DEPS>
std::enable_if_t<!std::is_same_v<T, bool>, bool> has_arg(const boost::program_options::variables_map& vm, const arg_descriptor<T, required, dependent, NUM_DEPS>& arg)
{
std::enable_if_t<!std::is_same_v<T, bool>, bool> has_arg(
const boost::program_options::variables_map& vm,
const arg_descriptor<T, required, dependent, NUM_DEPS>& arg) {
auto value = vm[arg.name];
return !value.empty();
}
template <typename T, bool required, bool dependent, int NUM_DEPS>
bool is_arg_defaulted(const boost::program_options::variables_map& vm, const arg_descriptor<T, required, dependent, NUM_DEPS>& arg)
{
bool is_arg_defaulted(
const boost::program_options::variables_map& vm,
const arg_descriptor<T, required, dependent, NUM_DEPS>& arg) {
return vm[arg.name].defaulted();
}
template <typename T>
T get_arg(const boost::program_options::variables_map& vm, const arg_descriptor<T, false, true>& arg)
{
T get_arg(
const boost::program_options::variables_map& vm,
const arg_descriptor<T, false, true>& arg) {
return arg.depf(get_arg(vm, arg.ref), is_arg_defaulted(vm, arg), vm[arg.name].template as<T>());
}
template <typename T, int NUM_DEPS>
T get_arg(const boost::program_options::variables_map& vm, const arg_descriptor<T, false, true, NUM_DEPS>& arg)
{
T get_arg(
const boost::program_options::variables_map& vm,
const arg_descriptor<T, false, true, NUM_DEPS>& arg) {
std::array<bool, NUM_DEPS> depval;
for (size_t i = 0; i < depval.size(); ++i)
depval[i] = get_arg(vm, *arg.ref[i]);
@ -286,18 +301,17 @@ namespace command_line
}
template <typename T, bool required>
T get_arg(const boost::program_options::variables_map& vm, const arg_descriptor<T, required>& arg)
{
T get_arg(const boost::program_options::variables_map& vm, const arg_descriptor<T, required>& arg) {
return vm[arg.name].template as<T>();
}
template <bool dependent, int NUM_DEPS>
inline bool has_arg(const boost::program_options::variables_map& vm, const arg_descriptor<bool, false, dependent, NUM_DEPS>& arg)
{
inline bool has_arg(
const boost::program_options::variables_map& vm,
const arg_descriptor<bool, false, dependent, NUM_DEPS>& arg) {
return get_arg(vm, arg);
}
extern const arg_descriptor<bool> arg_help;
extern const arg_descriptor<bool> arg_version;
@ -312,4 +326,4 @@ namespace command_line
// Clears the screen using readline, if available, otherwise trying some terminal escape hacks.
void clear_screen();
}
} // namespace command_line

5
src/common/common_fwd.h
View file

@ -30,10 +30,9 @@
#pragma once
namespace tools
{
namespace tools {
struct login;
class password_container;
class t_http_connection;
class threadpool;
}
} // namespace tools

39
src/common/compat/glibc_compat.cpp
View file

@ -1,10 +1,11 @@
#include <fnmatch.h>
#include <glob.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <cstddef>
#include <cstdint>
#include <strings.h>
#include <string.h>
#include <glob.h>
#include <unistd.h>
#include <fnmatch.h>
#if defined(HAVE_SYS_SELECT_H)
#include <sys/select.h>
@ -13,8 +14,7 @@
// Prior to GLIBC_2.14, memcpy was aliased to memmove.
extern "C" void* memmove(void* a, const void* b, size_t c);
// extern "C" void* memset(void* a, int b, long unsigned int c);
extern "C" void* memcpy(void* a, const void* b, size_t c)
{
extern "C" void* memcpy(void* a, const void* b, size_t c) {
return memmove(a, b, c);
}
@ -24,8 +24,7 @@ extern "C" void __chk_fail(void) __attribute__((__noreturn__));
extern "C" int64_t __udivmoddi4(uint64_t u, uint64_t v, uint64_t* rp);
extern "C" int64_t __wrap___divmoddi4(int64_t u, int64_t v, int64_t* rp)
{
extern "C" int64_t __wrap___divmoddi4(int64_t u, int64_t v, int64_t* rp) {
int32_t c1 = 0, c2 = 0;
int64_t uu = u, vv = v;
int64_t w;
@ -54,18 +53,14 @@ extern "C" int64_t __wrap___divmoddi4(int64_t u, int64_t v, int64_t* rp)
#undef explicit_bzero
/* Set LEN bytes of S to 0. The compiler will not delete a call to
this function, even if S is dead after the call. */
void
explicit_bzero (void *s, size_t len)
{
void explicit_bzero(void* s, size_t len) {
memset(s, '\0', len);
/* Compiler barrier. */
asm volatile("" ::: "memory");
}
// Redefine explicit_bzero_chk
void
__explicit_bzero_chk (void *dst, size_t len, size_t dstlen)
{
void __explicit_bzero_chk(void* dst, size_t len, size_t dstlen) {
/* Inline __memset_chk to avoid a PLT reference to __memset_chk. */
if (__glibc_unlikely(dstlen < len))
__chk_fail();
@ -80,7 +75,11 @@ __explicit_bzero_chk (void *dst, size_t len, size_t dstlen)
#define strong_alias (__explicit_bzero_chk, __explicit_bzero_chk_internal)
#undef glob
extern "C" int glob_old(const char * pattern, int flags, int (*errfunc) (const char *epath, int eerrno), glob_t *pglob);
extern "C" int glob_old(
const char* pattern,
int flags,
int (*errfunc)(const char* epath, int eerrno),
glob_t* pglob);
#ifdef __i386__
__asm__(".symver glob_old,glob@GLIBC_2.0");
#elif defined(__amd64__)
@ -91,8 +90,10 @@ __asm(".symver glob_old,glob@GLIBC_2.4");
__asm__(".symver glob_old,glob@GLIBC_2.17");
#endif
extern "C" int __wrap_glob(const char * pattern, int flags, int (*errfunc) (const char *epath, int eerrno), glob_t *pglob)
{
extern "C" int __wrap_glob(
const char* pattern,
int flags,
int (*errfunc)(const char* epath, int eerrno),
glob_t* pglob) {
return glob_old(pattern, flags, errfunc, pglob);
}

41
src/common/error.cpp
View file

@ -28,49 +28,36 @@
#include <string>
namespace
{
struct category final : std::error_category
{
virtual const char* name() const noexcept override final
{
return "common_category()";
}
namespace {
struct category final : std::error_category {
virtual const char* name() const noexcept override final { return "common_category()"; }
virtual std::string message(int value) const override final
{
switch (common_error(value))
{
virtual std::string message(int value) const override final {
switch (common_error(value)) {
case common_error::kInvalidArgument:
return std::error_code{static_cast<int>(std::errc::invalid_argument), *this}.message();
return std::error_code{static_cast<int>(std::errc::invalid_argument), *this}
.message();
case common_error::kInvalidErrorCode:
return "expect<T> was given an error value of zero";
default:
break;
default: break;
}
return "Unknown basic_category() value";
}
virtual std::error_condition default_error_condition(int value) const noexcept override final
{
virtual std::error_condition default_error_condition(int value) const noexcept override final {
// maps specific errors to generic `std::errc` cases.
switch (common_error(value))
{
switch (common_error(value)) {
case common_error::kInvalidArgument:
case common_error::kInvalidErrorCode:
return std::errc::invalid_argument;
default:
break;
case common_error::kInvalidErrorCode: return std::errc::invalid_argument;
default: break;
}
return std::error_condition{value, *this};
}
};
const category instance{};
}
} // namespace
std::error_category const& common_category() noexcept
{
std::error_category const& common_category() noexcept {
return instance;
}

15
src/common/error.h
View file

@ -28,8 +28,7 @@
#include <system_error>
enum class common_error : int
{
enum class common_error : int {
// 0 is reserved for no error, as per expect<T>
kInvalidArgument = 1, //!< A function argument is invalid
kInvalidErrorCode //!< Default `std::error_code` given to `expect<T>`
@ -37,15 +36,11 @@ enum class common_error : int
std::error_category const& common_category() noexcept;
inline std::error_code make_error_code(::common_error value) noexcept
{
inline std::error_code make_error_code(::common_error value) noexcept {
return std::error_code{static_cast<int>(value), common_category()};
}
namespace std
{
namespace std {
template <>
struct is_error_code_enum<::common_error>
: true_type
{};
}
struct is_error_code_enum<::common_error> : true_type {};
} // namespace std

25
src/common/expect.cpp
View file

@ -26,25 +26,21 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "expect.h"
#include "common/fs.h"
#include <string>
namespace detail
{
namespace
{
std::string generate_error(const char* msg, const char* file, unsigned line)
{
#include "common/fs.h"
namespace detail {
namespace {
std::string generate_error(const char* msg, const char* file, unsigned line) {
std::string error_msg{};
if (msg)
{
if (msg) {
error_msg.append(msg);
if (file)
error_msg.append(" (");
}
if (file)
{
if (file) {
error_msg.append("thrown at ");
// remove path, get just filename + extension
@ -57,12 +53,11 @@ namespace detail
error_msg.push_back(')');
return error_msg;
}
}
} // namespace
void expect::throw_(std::error_code ec, const char* msg, const char* file, unsigned line)
{
void expect::throw_(std::error_code ec, const char* msg, const char* file, unsigned line) {
if (msg || file)
throw std::system_error{ec, generate_error(msg, file, line)};
throw std::system_error{ec};
}
} // detail
} // namespace detail

190
src/common/expect.h
View file

@ -37,16 +37,14 @@
//! If precondition fails, return `::error::kInvalidArgument` in current scope.
#define MONERO_PRECOND(...) \
do \
{ \
do { \
if (!(__VA_ARGS__)) \
return {::common_error::kInvalidArgument}; \
} while (0)
//! Check `expect<void>` and return errors in current scope.
#define MONERO_CHECK(...) \
do \
{ \
do { \
const ::expect<void> result = __VA_ARGS__; \
if (!result) \
return result.error(); \
@ -57,38 +55,34 @@
\throw std::system_error with `expect<T>::error()`, filename and line
number when `expect<T>::has_error() == true`.*/
#define MONERO_UNWRAP(...) \
::detail::expect::unwrap( __VA_ARGS__ , nullptr, __FILE__ , __LINE__ )
#define MONERO_UNWRAP(...) ::detail::expect::unwrap(__VA_ARGS__, nullptr, __FILE__, __LINE__)
/* \throw std::system_error with `code` and `msg` as part of the details. The
filename and line number will automatically be injected into the explanation
string. `code` can be any enum convertible to `std::error_code`. */
#define MONERO_THROW(code, msg) \
::detail::expect::throw_( code , msg , __FILE__ , __LINE__ )
#define MONERO_THROW(code, msg) ::detail::expect::throw_(code, msg, __FILE__, __LINE__)
template <typename>
class expect;
template<typename> class expect;
namespace detail
{
struct expect
{
namespace detail {
struct expect {
//! \throw std::system_error with `ec`, optional `msg` and/or optional `file` + `line`.
static void throw_(std::error_code ec, const char* msg, const char* file, unsigned line);
//! If `result.has_error()` call `throw_`. Otherwise, \return `*result` by move.
template <typename T>
static T unwrap(::expect<T>&& result, const char* error_msg, const char* file, unsigned line)
{
static T unwrap(::expect<T>&& result, const char* error_msg, const char* file, unsigned line) {
if (!result)
throw_(result.error(), error_msg, file, line);
return std::move(*result);
}
//! If `result.has_error()` call `throw_`.
static void unwrap(::expect<void>&& result, const char* error_msg, const char* file, unsigned line);
static void unwrap(
::expect<void>&& result, const char* error_msg, const char* file, unsigned line);
};
}
} // namespace detail
/*!
`expect<T>` is a value or error implementation, similar to Rust std::result
@ -105,8 +99,8 @@ namespace detail
types have a `operator==` defined between them (i.e.
`assert(expect<int>{100} == expect<short>{100});`). Comparisons can also be
done against `std::error_code` objects or error code enums directly (i.e.
`assert(expect<int>{make_error_code(common_error::kInvalidArgument)} == error::kInvalidArgument)`).
Comparison of default constructed `std::error_code` will always fail.
`assert(expect<int>{make_error_code(common_error::kInvalidArgument)} ==
error::kInvalidArgument)`). Comparison of default constructed `std::error_code` will always fail.
"Generic" comparisons can be done with `std::error_condition` via the `matches`
method only (i.e.
`assert(expect<int>{make_error_code{common_error::kInvalidErrorCode}.matches(std::errc::invalid_argument))`),
@ -125,15 +119,12 @@ namespace detail
\note See `src/common/error.h` for creating a custom error enum.
*/
template <typename T>
class expect
{
class expect {
static_assert(std::is_nothrow_destructible<T>(), "T must have a nothrow destructor");
template <typename U>
static constexpr bool is_convertible() noexcept
{
return std::is_constructible<T, U>() &&
std::is_convertible<U, T>();
static constexpr bool is_convertible() noexcept {
return std::is_constructible<T, U>() && std::is_convertible<U, T>();
}
// MEMBERS
@ -141,27 +132,23 @@ class expect
std::aligned_storage_t<sizeof(T), alignof(T)> storage_;
// MEMBERS
T& get() noexcept
{
T& get() noexcept {
assert(has_value());
return *reinterpret_cast<T*>(std::addressof(storage_));
}
T const& get() const noexcept
{
T const& get() const noexcept {
assert(has_value());
return *reinterpret_cast<T const*>(std::addressof(storage_));
}
template <typename U>
void store(U&& value) noexcept(std::is_nothrow_constructible<T, U>())
{
void store(U&& value) noexcept(std::is_nothrow_constructible<T, U>()) {
new (std::addressof(storage_)) T{std::forward<U>(value)};
code_ = std::error_code{};
}
void maybe_throw() const
{
void maybe_throw() const {
if (has_error())
::detail::expect::throw_(error(), nullptr, nullptr, 0);
}
@ -175,62 +162,52 @@ public:
/*! Store an error, `code`, in the `expect` object. If `code` creates a
`std::error_code` object whose `.value() == 0`, then `error()` will be set
to `::common_error::kInvalidErrorCode`. */
expect(std::error_code const& code) noexcept
: code_(code), storage_()
{
expect(std::error_code const& code) noexcept : code_(code), storage_() {
if (!has_error())
code_ = ::common_error::kInvalidErrorCode;
}
//! Store a value, `val`, in the `expect` object.
expect(T val) noexcept(std::is_nothrow_move_constructible<T>())
: code_(), storage_()
{
expect(T val) noexcept(std::is_nothrow_move_constructible<T>()) : code_(), storage_() {
store(std::move(val));
}
expect(expect const& src) noexcept(std::is_nothrow_copy_constructible<T>())
: code_(src.error()), storage_()
{
expect(expect const& src) noexcept(std::is_nothrow_copy_constructible<T>()) :
code_(src.error()), storage_() {
if (src.has_value())
store(src.get());
}
//! Copy conversion from `U` to `T`.
template <typename U, typename = std::enable_if_t<is_convertible<U const&>()>>
expect(expect<U> const& src) noexcept(std::is_nothrow_constructible<T, U const&>())
: code_(src.error()), storage_()
{
expect(expect<U> const& src) noexcept(std::is_nothrow_constructible<T, U const&>()) :
code_(src.error()), storage_() {
if (src.has_value())
store(*src);
}
expect(expect&& src) noexcept(std::is_nothrow_move_constructible<T>())
: code_(src.error()), storage_()
{
expect(expect&& src) noexcept(std::is_nothrow_move_constructible<T>()) :
code_(src.error()), storage_() {
if (src.has_value())
store(std::move(src.get()));
}
//! Move conversion from `U` to `T`.
template <typename U, typename = std::enable_if_t<is_convertible<U>()>>
expect(expect<U>&& src) noexcept(std::is_nothrow_constructible<T, U>())
: code_(src.error()), storage_()
{
expect(expect<U>&& src) noexcept(std::is_nothrow_constructible<T, U>()) :
code_(src.error()), storage_() {
if (src.has_value())
store(std::move(*src));
}
~expect() noexcept
{
~expect() noexcept {
if (has_value())
get().~T();
}
expect& operator=(expect const& src) noexcept(std::is_nothrow_copy_constructible<T>() && std::is_nothrow_copy_assignable<T>())
{
if (this != std::addressof(src))
{
expect& operator=(expect const& src) noexcept(
std::is_nothrow_copy_constructible<T>() && std::is_nothrow_copy_assignable<T>()) {
if (this != std::addressof(src)) {
if (has_value() && src.has_value())
get() = src.get();
else if (has_value())
@ -244,10 +221,9 @@ public:
/*! Move `src` into `this`. If `src.has_value() && addressof(src) != this`
then `src.value() will be in a "moved from state". */
expect& operator=(expect&& src) noexcept(std::is_nothrow_move_constructible<T>() && std::is_nothrow_move_assignable<T>())
{
if (this != std::addressof(src))
{
expect& operator=(expect&& src) noexcept(
std::is_nothrow_move_constructible<T>() && std::is_nothrow_move_assignable<T>()) {
if (this != std::addressof(src)) {
if (has_value() && src.has_value())
get() = std::move(src.get());
else if (has_value())
@ -272,23 +248,20 @@ public:
std::error_code error() const noexcept { return code_; }
//! \return Value if `has_value()` otherwise \throw `std::system_error{error()}`.
T& value() &
{
T& value() & {
maybe_throw();
return get();
}
//! \return Value if `has_value()` otherwise \throw `std::system_error{error()}`.
T const& value() const &
{
T const& value() const& {
maybe_throw();
return get();
}
/*! Same as other overloads, but expressions such as `foo(bar().value())`
will automatically perform moves with no copies. */
T&& value() &&
{
T&& value() && {
maybe_throw();
return std::move(get());
}
@ -307,38 +280,30 @@ public:
\return True if `has_value() == rhs.has_value()` and if values or errors are equal.
*/
template <typename U>
bool equal(expect<U> const& rhs) const noexcept(noexcept(*std::declval<expect<T>>() == *rhs))
{
return has_value() && rhs.has_value() ?
get() == *rhs : error() == rhs.error();
bool equal(expect<U> const& rhs) const noexcept(noexcept(*std::declval<expect<T>>() == *rhs)) {
return has_value() && rhs.has_value() ? get() == *rhs : error() == rhs.error();
}
//! \return False if `has_value()`, otherwise `error() == rhs`.
bool equal(std::error_code const& rhs) const noexcept
{
return has_error() && error() == rhs;
}
bool equal(std::error_code const& rhs) const noexcept { return has_error() && error() == rhs; }
/*!
\note This function is `noexcept` when `U == T` is `noexcept`.
\return False if `has_error()`, otherwise `value() == rhs`.
*/
template <typename U, typename = std::enable_if_t<!std::is_constructible_v<std::error_code, U>>>
bool equal(U const& rhs) const noexcept(noexcept(*std::declval<expect<T>>() == rhs))
{
bool equal(U const& rhs) const noexcept(noexcept(*std::declval<expect<T>>() == rhs)) {
return has_value() && get() == rhs;
}
//! \return False if `has_value()`, otherwise `error() == rhs`.
bool matches(std::error_condition const& rhs) const noexcept
{
bool matches(std::error_condition const& rhs) const noexcept {
return has_error() && error() == rhs;
}
};
template <>
class expect<void>
{
class expect<void> {
std::error_code code_;
public:
@ -346,13 +311,9 @@ public:
using error_type = std::error_code;
//! Create a successful object.
expect() noexcept
: code_()
{}
expect() noexcept : code_() {}
expect(std::error_code const& code) noexcept
: code_(code)
{
expect(std::error_code const& code) noexcept : code_(code) {
if (!has_error())
code_ = ::common_error::kInvalidErrorCode;
}
@ -371,75 +332,58 @@ public:
std::error_code error() const noexcept { return code_; }
//! \return `error() == rhs.error()`.
bool equal(expect const& rhs) const noexcept
{
return error() == rhs.error();
}
bool equal(expect const& rhs) const noexcept { return error() == rhs.error(); }
//! \return `has_error() && error() == rhs`.
bool equal(std::error_code const& rhs) const noexcept
{
return has_error() && error() == rhs;
}
bool equal(std::error_code const& rhs) const noexcept { return has_error() && error() == rhs; }
//! \return False if `has_value()`, otherwise `error() == rhs`.
bool matches(std::error_condition const& rhs) const noexcept
{
bool matches(std::error_condition const& rhs) const noexcept {
return has_error() && error() == rhs;
}
};
//! \return An `expect<void>` object with `!has_error()`.
inline expect<void> success() noexcept { return expect<void>{}; }
inline expect<void> success() noexcept {
return expect<void>{};
}
template <typename T, typename U>
inline
bool operator==(expect<T> const& lhs, expect<U> const& rhs) noexcept(noexcept(lhs.equal(rhs)))
{
inline bool operator==(expect<T> const& lhs, expect<U> const& rhs) noexcept(
noexcept(lhs.equal(rhs))) {
return lhs.equal(rhs);
}
template <typename T, typename U>
inline
bool operator==(expect<T> const& lhs, U const& rhs) noexcept(noexcept(lhs.equal(rhs)))
{
inline bool operator==(expect<T> const& lhs, U const& rhs) noexcept(noexcept(lhs.equal(rhs))) {
return lhs.equal(rhs);
}
template <typename T, typename U>
inline
bool operator==(T const& lhs, expect<U> const& rhs) noexcept(noexcept(rhs.equal(lhs)))
{
inline bool operator==(T const& lhs, expect<U> const& rhs) noexcept(noexcept(rhs.equal(lhs))) {
return rhs.equal(lhs);
}
template <typename T, typename U>
inline
bool operator!=(expect<T> const& lhs, expect<U> const& rhs) noexcept(noexcept(lhs.equal(rhs)))
{
inline bool operator!=(expect<T> const& lhs, expect<U> const& rhs) noexcept(
noexcept(lhs.equal(rhs))) {
return !lhs.equal(rhs);
}
template <typename T, typename U>
inline
bool operator!=(expect<T> const& lhs, U const& rhs) noexcept(noexcept(lhs.equal(rhs)))
{
inline bool operator!=(expect<T> const& lhs, U const& rhs) noexcept(noexcept(lhs.equal(rhs))) {
return !lhs.equal(rhs);
}
template <typename T, typename U>
inline
bool operator!=(T const& lhs, expect<U> const& rhs) noexcept(noexcept(rhs.equal(lhs)))
{
inline bool operator!=(T const& lhs, expect<U> const& rhs) noexcept(noexcept(rhs.equal(lhs))) {
return !rhs.equal(lhs);
}
namespace detail
{
inline void expect::unwrap(::expect<void>&& result, const char* error_msg, const char* file, unsigned line)
{
namespace detail {
inline void expect::unwrap(
::expect<void>&& result, const char* error_msg, const char* file, unsigned line) {
if (!result)
throw_(result.error(), error_msg, file, line);
}
}
} // namespace detail

170
src/common/file.cpp
View file

@ -30,33 +30,37 @@
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
//
#include "file.h"
#include "fs-format.h"
#include "logging/oxen_logger.h"
#include <unistd.h>
#include <cstdio>
#include <fstream>
#include "fs-format.h"
#include "logging/oxen_logger.h"
#ifdef WIN32
#include "epee/string_tools.h"
#ifndef STRSAFE_NO_DEPRECATE
#define STRSAFE_NO_DEPRECATE
#endif
#include <windows.h>
#include <shlobj.h>
#include <strsafe.h>
#include <windows.h>
#else
#include <sys/file.h>
#include <sys/utsname.h>
#include <sys/stat.h>
#include <sys/utsname.h>
#endif
#ifdef __GLIBC__
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h>
#include <cstring>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <cctype>
#include <cstring>
#endif
#include "cryptonote_config.h"
@ -66,8 +70,7 @@ namespace tools {
static auto logcat = log::Cat("util");
#ifndef _WIN32
static int flock_exnb(int fd)
{
static int flock_exnb(int fd) {
struct flock fl;
int ret;
@ -85,24 +88,20 @@ namespace tools {
private_file::private_file() noexcept : m_handle(), m_filename() {}
private_file::private_file(std::FILE* handle, fs::path filename) noexcept
: m_handle(handle), m_filename(std::move(filename)) {}
private_file::private_file(std::FILE* handle, fs::path filename) noexcept :
m_handle(handle), m_filename(std::move(filename)) {}
private_file private_file::create(fs::path name)
{
private_file private_file::create(fs::path name) {
#ifdef WIN32
struct close_handle
{
void operator()(HANDLE handle) const noexcept
{
CloseHandle(handle);
}
struct close_handle {
void operator()(HANDLE handle) const noexcept { CloseHandle(handle); }
};
std::unique_ptr<void, close_handle> process = nullptr;
{
HANDLE temp{};
const bool fail = OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, std::addressof(temp)) == 0;
const bool fail =
OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, std::addressof(temp)) == 0;
process.reset(temp);
if (fail)
return {};
@ -114,7 +113,8 @@ namespace tools {
return {};
std::unique_ptr<char[]> sid{new char[sid_size]};
if (!GetTokenInformation(process.get(), TokenOwner, sid.get(), sid_size, std::addressof(sid_size)))
if (!GetTokenInformation(
process.get(), TokenOwner, sid.get(), sid_size, std::addressof(sid_size)))
return {};
const PSID psid = reinterpret_cast<const PTOKEN_OWNER>(sid.get())->Owner;
@ -125,35 +125,36 @@ namespace tools {
if (!InitializeAcl(reinterpret_cast<PACL>(dacl.get()), daclSize, ACL_REVISION))
return {};
if (!AddAccessAllowedAce(reinterpret_cast<PACL>(dacl.get()), ACL_REVISION, (READ_CONTROL | FILE_GENERIC_READ | DELETE), psid))
if (!AddAccessAllowedAce(
reinterpret_cast<PACL>(dacl.get()),
ACL_REVISION,
(READ_CONTROL | FILE_GENERIC_READ | DELETE),
psid))
return {};
SECURITY_DESCRIPTOR descriptor{};
if (!InitializeSecurityDescriptor(std::addressof(descriptor), SECURITY_DESCRIPTOR_REVISION))
return {};
if (!SetSecurityDescriptorDacl(std::addressof(descriptor), true, reinterpret_cast<PACL>(dacl.get()), false))
if (!SetSecurityDescriptorDacl(
std::addressof(descriptor), true, reinterpret_cast<PACL>(dacl.get()), false))
return {};
SECURITY_ATTRIBUTES attributes{sizeof(SECURITY_ATTRIBUTES), std::addressof(descriptor), false};
std::unique_ptr<void, close_handle> file{
CreateFileW(
std::unique_ptr<void, close_handle> file{CreateFileW(
name.c_str(),
GENERIC_WRITE, FILE_SHARE_READ,
GENERIC_WRITE,
FILE_SHARE_READ,
std::addressof(attributes),
CREATE_NEW, (FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE),
nullptr
)
};
if (file)
{
CREATE_NEW,
(FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE),
nullptr)};
if (file) {
const int fd = _open_osfhandle(reinterpret_cast<intptr_t>(file.get()), 0);
if (0 <= fd)
{
if (0 <= fd) {
file.release();
std::FILE* real_file = _fdopen(fd, "w");
if (!real_file)
{
if (!real_file) {
_close(fd);
}
return {real_file, std::move(name)};
@ -161,11 +162,9 @@ namespace tools {
}
#else
const int fdr = open(name.c_str(), (O_RDONLY | O_CREAT), S_IRUSR);
if (0 <= fdr)
{
if (0 <= fdr) {
struct stat rstats = {};
if (fstat(fdr, std::addressof(rstats)) != 0)
{
if (fstat(fdr, std::addressof(rstats)) != 0) {
close(fdr);
return {};
}
@ -174,15 +173,13 @@ namespace tools {
fchmod(fdr, rstats.st_mode);
close(fdr);
if (0 <= fdw)
{
if (0 <= fdw) {
struct stat wstats = {};
if (fstat(fdw, std::addressof(wstats)) == 0 &&
rstats.st_dev == wstats.st_dev && rstats.st_ino == wstats.st_ino &&
flock_exnb(fdw) == 0 && ftruncate(fdw, 0) == 0)
{
if (fstat(fdw, std::addressof(wstats)) == 0 && rstats.st_dev == wstats.st_dev &&
rstats.st_ino == wstats.st_ino && flock_exnb(fdw) == 0 && ftruncate(fdw, 0) == 0) {
std::FILE* file = fdopen(fdw, "w");
if (file) return {file, std::move(name)};
if (file)
return {file, std::move(name)};
}
close(fdw);
}
@ -191,53 +188,50 @@ namespace tools {
return {};
}
private_file::~private_file() noexcept
{
private_file::~private_file() noexcept {
std::error_code ignored;
fs::remove(filename(), ignored);
}
file_locker::file_locker(const fs::path& filename)
{
file_locker::file_locker(const fs::path& filename) {
#ifdef WIN32
m_fd = INVALID_HANDLE_VALUE;
m_fd = CreateFileW(filename.c_str(), GENERIC_READ, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if (m_fd != INVALID_HANDLE_VALUE)
{
m_fd = CreateFileW(
filename.c_str(), GENERIC_READ, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
if (m_fd != INVALID_HANDLE_VALUE) {
OVERLAPPED ov;
memset(&ov, 0, sizeof(ov));
if (!LockFileEx(m_fd, LOCKFILE_FAIL_IMMEDIATELY | LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &ov))
{
log::error(logcat, "Failed to lock {}: {}", filename, std::error_code(GetLastError(), std::system_category()).message());
if (!LockFileEx(m_fd, LOCKFILE_FAIL_IMMEDIATELY | LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &ov)) {
log::error(
logcat,
"Failed to lock {}: {}",
filename,
std::error_code(GetLastError(), std::system_category()).message());
CloseHandle(m_fd);
m_fd = INVALID_HANDLE_VALUE;
}
}
else
{
log::error(logcat, "Failed to open {}: {}", filename, std::error_code(GetLastError(), std::system_category()).message());
} else {
log::error(
logcat,
"Failed to open {}: {}",
filename,
std::error_code(GetLastError(), std::system_category()).message());
}
#else
m_fd = open(filename.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666);
if (m_fd != -1)
{
if (flock_exnb(m_fd) == -1)
{
if (m_fd != -1) {
if (flock_exnb(m_fd) == -1) {
log::error(logcat, "Failed to lock {}: {}", filename, std::strerror(errno));
close(m_fd);
m_fd = -1;
}
}
else
{
} else {
log::error(logcat, "Failed to open {}: {}", filename, std::strerror(errno));
}
#endif
}
file_locker::~file_locker()
{
if (locked())
{
file_locker::~file_locker() {
if (locked()) {
#ifdef WIN32
CloseHandle(m_fd);
#else
@ -245,8 +239,7 @@ namespace tools {
#endif
}
}
bool file_locker::locked() const
{
bool file_locker::locked() const {
#ifdef WIN32
return m_fd != INVALID_HANDLE_VALUE;
#else
@ -254,14 +247,11 @@ namespace tools {
#endif
}
#ifdef _WIN32
fs::path get_special_folder_path(int nfolder, bool iscreate)
{
fs::path get_special_folder_path(int nfolder, bool iscreate) {
WCHAR psz_path[MAX_PATH] = L"";
if (SHGetSpecialFolderPathW(NULL, psz_path, nfolder, iscreate))
{
if (SHGetSpecialFolderPathW(NULL, psz_path, nfolder, iscreate)) {
return fs::path{psz_path};
}
@ -282,17 +272,14 @@ namespace tools {
#endif
}
fs::path get_default_data_dir()
{
fs::path get_default_data_dir() {
return get_default_parent_dir() / fs::u8path(cryptonote::DATA_DIRNAME);
}
fs::path get_depreciated_default_data_dir()
{
fs::path get_depreciated_default_data_dir() {
return get_default_parent_dir() / fs::u8path(cryptonote::old::DATA_DIRNAME);
}
void set_strict_default_file_permissions(bool strict)
{
void set_strict_default_file_permissions(bool strict) {
#if defined(__MINGW32__) || defined(__MINGW__)
// no clue about the odd one out
#else
@ -301,12 +288,12 @@ namespace tools {
#endif
}
bool slurp_file(const fs::path& filename, std::string& contents)
{
bool slurp_file(const fs::path& filename, std::string& contents) {
try {
fs::ifstream in(filename, std::ios::binary);
std::string content((std::istreambuf_iterator<char>(in)), (std::istreambuf_iterator<char>()));
std::string content(
(std::istreambuf_iterator<char>(in)), (std::istreambuf_iterator<char>()));
contents = std::move(content);
return true;
} catch (...) {
@ -314,8 +301,7 @@ namespace tools {
}
}
bool dump_file(const fs::path& filename, std::string_view contents)
{
bool dump_file(const fs::path& filename, std::string_view contents) {
fs::ofstream out;
out.exceptions(std::ifstream::failbit | std::ifstream::badbit);
try {
@ -327,4 +313,4 @@ namespace tools {
}
}
}
} // namespace tools

21
src/common/file.h
View file

@ -31,18 +31,18 @@
//
#pragma once
#include <string>
#include <cstdio>
#include <memory>
#include <optional>
#include <string>
#include <system_error>
#include "fs.h"
#ifdef _WIN32
#include "windows.h"
#endif
/**
* Various utilities for dealing with files/directories.
*/
@ -50,12 +50,9 @@
namespace tools {
//! Functional class for closing C file handles.
struct close_file
{
void operator()(std::FILE* handle) const noexcept
{
if (handle)
{
struct close_file {
void operator()(std::FILE* handle) const noexcept {
if (handle) {
std::fclose(handle);
}
}
@ -67,8 +64,8 @@ namespace tools {
fs::path m_filename;
private_file(std::FILE* handle, fs::path filename) noexcept;
public:
public:
//! `handle() == nullptr && filename.empty()`.
private_file() noexcept;
@ -86,12 +83,12 @@ namespace tools {
const fs::path& filename() const noexcept { return m_filename; }
};
class file_locker
{
class file_locker {
public:
file_locker(const fs::path& filename);
~file_locker();
bool locked() const;
private:
#ifdef WIN32
HANDLE m_fd;
@ -135,4 +132,4 @@ namespace tools {
/// Dumps (binary) string contents to disk. The file is overwritten if it already exists.
bool dump_file(const fs::path& filename, std::string_view contents);
}
} // namespace tools

18
src/common/formattable.h
View file

@ -1,6 +1,7 @@
#pragma once
#include <fmt/format.h>
#include <string_view>
#include <type_traits>
#include <utility>
@ -48,14 +49,14 @@ namespace formattable {
constexpr bool has_to_hex_string_method = false;
template <typename T>
inline constexpr bool has_to_string_method<T,
std::void_t<decltype(std::declval<const T&>().to_string())>
> = true;
inline constexpr bool
has_to_string_method<T, std::void_t<decltype(std::declval<const T&>().to_string())>> =
true;
template <typename T>
inline constexpr bool has_to_hex_string_method<T,
std::void_t<decltype(std::declval<const T&>().to_hex_string())>
> = true;
inline constexpr bool has_to_hex_string_method<
T,
std::void_t<decltype(std::declval<const T&>().to_hex_string())>> = true;
} // namespace detail
@ -86,7 +87,8 @@ namespace formattable {
#ifdef __cpp_lib_is_scoped_enum // C++23
static_assert(std::is_scoped_enum_v<T>);
#else
static_assert(std::is_enum_v<T> && !std::is_convertible_v<T, std::underlying_type_t<T>>,
static_assert(
std::is_enum_v<T> && !std::is_convertible_v<T, std::underlying_type_t<T>>,
"formattable::via_underlying<T> type is not a scoped enum");
#endif
template <typename FormatContext>
@ -98,8 +100,6 @@ namespace formattable {
} // namespace formattable
namespace fmt {
template <typename T, typename Char>

5
src/common/fs-format.h
View file

@ -9,12 +9,11 @@
namespace fmt {
template <>
struct formatter<ghc::filesystem::path> : formatter<std::string>
{
struct formatter<ghc::filesystem::path> : formatter<std::string> {
template <typename FormatContext>
auto format(const ghc::filesystem::path& val, FormatContext& ctx) const {
return formatter<std::string>::format(val.u8string(), ctx);
}
};
}
} // namespace fmt
#endif

2
src/common/fs.h
View file

@ -16,7 +16,7 @@ namespace fs {
using ifstream = std::ifstream;
using ofstream = std::ofstream;
using fstream = std::fstream;
}
} // namespace fs
#else
#include <ghc/filesystem.hpp>

21
src/common/hex.h
View file

@ -1,15 +1,19 @@
#pragma once
#include <oxenc/hex.h>
#include <type_traits>
#include "epee/span.h" // epee
namespace tools {
// Reads a hex string directly into a trivially copyable type T without performing any temporary
// allocation. Returns false if the given string is not hex or does not match T in length,
// otherwise copies directly into `x` and returns true.
template <typename T, typename = std::enable_if_t<
!std::is_const_v<T> && (std::is_trivially_copyable_v<T> || epee::is_byte_spannable<T>)
>>
template <
typename T,
typename = std::enable_if_t<
!std::is_const_v<T> &&
(std::is_trivially_copyable_v<T> || epee::is_byte_spannable<T>)>>
bool hex_to_type(std::string_view hex, T& x) {
if (!oxenc::is_hex(hex) || hex.size() != 2 * sizeof(T))
return false;
@ -18,11 +22,12 @@ namespace tools {
}
/// Converts a standard layout, padding-free type into a hex string of its contents.
template <typename T, typename = std::enable_if_t<
(std::is_standard_layout_v<T> && std::has_unique_object_representations_v<T>)
|| epee::is_byte_spannable<T>
>>
template <
typename T,
typename = std::enable_if_t<
(std::is_standard_layout_v<T> && std::has_unique_object_representations_v<T>) ||
epee::is_byte_spannable<T>>>
std::string type_to_hex(const T& val) {
return oxenc::to_hex(std::string_view{reinterpret_cast<const char*>(&val), sizeof(val)});
}
}
} // namespace tools

73
src/common/i18n.cpp
View file

@ -28,22 +28,41 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "i18n.h"
#include <cstdlib>
#include <cstring>
#include <logging/oxen_logger.h>
#include <algorithm>
#include <cctype>
#include <cstdint>
#include <string>
#include <cstdlib>
#include <cstring>
#include <map>
#include <string>
#include <utility>
#include <algorithm>
#include "file.h"
#include <logging/oxen_logger.h>
static auto logcat = oxen::log::Cat("i18n");
#define MAX_LANGUAGE_SIZE 16
static const unsigned char qm_magic[16] = {0x3c, 0xb8, 0x64, 0x18, 0xca, 0xef, 0x9c, 0x95, 0xcd, 0x21, 0x1c, 0xbf, 0x60, 0xa1, 0xbd, 0xdd};
static const unsigned char qm_magic[16] = {
0x3c,
0xb8,
0x64,
0x18,
0xca,
0xef,
0x9c,
0x95,
0xcd,
0x21,
0x1c,
0xbf,
0x60,
0xa1,
0xbd,
0xdd};
static std::map<std::string, std::string> i18n_entries;
@ -51,11 +70,11 @@ using namespace std::literals;
/* Logging isn't initialized yet when this is run */
/* add std::flush, because std::endl doesn't seem to flush, contrary to expected */
// #define i18n_log(x) do { std::cout << __FILE__ << ":" << __LINE__ << ": " << x << std::endl; std::cout << std::flush; } while(0)
// #define i18n_log(x) do { std::cout << __FILE__ << ":" << __LINE__ << ": " << x << std::endl;
// std::cout << std::flush; } while(0)
#define i18n_log(x) ((void)0)
std::string i18n_get_language()
{
std::string i18n_get_language() {
const char* e;
e = getenv("LANG");
@ -77,21 +96,18 @@ std::string i18n_get_language()
return "en";
std::transform(language.begin(), language.end(), language.begin(), tolower);
if (language.size() > MAX_LANGUAGE_SIZE)
{
if (language.size() > MAX_LANGUAGE_SIZE) {
i18n_log("Language from LANG/LC_ALL suspiciously long, defaulting to en");
return "en";
}
return language;
}
static uint32_t be32(const unsigned char *data)
{
static uint32_t be32(const unsigned char* data) {
return (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3];
}
static std::string utf16(const unsigned char *data, uint32_t len)
{
static std::string utf16(const unsigned char* data, uint32_t len) {
std::string s;
while (len >= 2) {
uint32_t code = (data[0] << 8) | data[1];
@ -107,17 +123,14 @@ static std::string utf16(const unsigned char *data, uint32_t len)
}
if (code <= 0x7f) {
s += (char)code;
}
else if (code <= 0x7ff) {
} else if (code <= 0x7ff) {
s += 0xc0 | (code >> 6);
s += 0x80 | (code & 0x3f);
}
else if (code <= 0xffff) {
} else if (code <= 0xffff) {
s += 0xe0 | (code >> 12);
s += 0x80 | ((code >> 6) & 0x3f);
s += 0x80 | (code & 0x3f);
}
else {
} else {
s += 0xf0 | (code >> 18);
s += 0x80 | ((code >> 12) & 0x3f);
s += 0x80 | ((code >> 6) & 0x3f);
@ -127,14 +140,12 @@ static std::string utf16(const unsigned char *data, uint32_t len)
return s;
}
static std::string utf8(const unsigned char *data, uint32_t len)
{
static std::string utf8(const unsigned char* data, uint32_t len) {
/* assume well formedness */
return std::string((const char*)data, len);
}
int i18n_set_language(const char *directory, const char *base, std::string language)
{
int i18n_set_language(const char* directory, const char* base, std::string language) {
i18n_log("i18n_set_language(" << directory << "," << base << ")");
if (!directory || !base)
return -1;
@ -157,7 +168,8 @@ int i18n_set_language(const char *directory, const char *base, std::string langu
i18n_log("Embedded translations file not found: " << basename);
const char* underscore = strchr(language.c_str(), '_');
if (underscore) {
std::string fallback_language = std::string(language, 0, underscore - language.c_str());
std::string fallback_language =
std::string(language, 0, underscore - language.c_str());
basename = base + "_"s + fallback_language + ".qm";
filename.replace_filename(fs::u8path(basename));
i18n_log("Loading translations for language " << fallback_language);
@ -246,9 +258,7 @@ int i18n_set_language(const char *directory, const char *base, std::string langu
i18n_log("Found messages at " << idx);
messages_idx = idx;
break;
default:
i18n_log("Found unsupported chunk type: " << chunk_type);
break;
default: i18n_log("Found unsupported chunk type: " << chunk_type); break;
}
idx += chunk_size;
@ -318,13 +328,10 @@ int i18n_set_language(const char *directory, const char *base, std::string langu
}
/* The entries is constant by that time */
const char *i18n_translate(const char *s, const std::string &context)
{
const char* i18n_translate(const char* s, const std::string& context) {
const std::string key = context + "\0"s + s;
std::map<std::string, std::string>::const_iterator i = i18n_entries.find(key);
if (i == i18n_entries.end())
return s;
return (*i).second.c_str();
}

10
src/common/i18n.h
View file

@ -32,7 +32,11 @@
#define QT_TRANSLATE_NOOP(context, str) i18n_translate(str, context)
std::string i18n_get_language();
int i18n_set_language(const char *directory, const char *base, std::string language = std::string());
int i18n_set_language(
const char* directory, const char* base, std::string language = std::string());
const char* i18n_translate(const char* str, const std::string& context);
inline const char *tr(const char *str) { return i18n_translate(str, std::string{}); }
bool find_embedded_file(const std::string &name, std::string &data); // In the generated translation_files.cpp
inline const char* tr(const char* str) {
return i18n_translate(str, std::string{});
}
bool find_embedded_file(
const std::string& name, std::string& data); // In the generated translation_files.cpp

16
src/common/json_binary_proxy.cpp
View file

@ -1,10 +1,12 @@
#include "json_binary_proxy.h"
#include <oxenc/hex.h>
#include <oxenc/base64.h>
#include <oxenc/hex.h>
namespace tools {
void load_binary_parameter_impl(std::string_view bytes, size_t raw_size, bool allow_raw, uint8_t* val_data) {
void load_binary_parameter_impl(
std::string_view bytes, size_t raw_size, bool allow_raw, uint8_t* val_data) {
if (allow_raw && bytes.size() == raw_size) {
std::memcpy(val_data, bytes.data(), bytes.size());
return;
@ -17,9 +19,11 @@ namespace tools {
const size_t b64_padded = (raw_size + 2) / 3 * 4;
const size_t b64_padding = raw_size % 3 == 1 ? 2 : raw_size % 3 == 2 ? 1 : 0;
const size_t b64_unpadded = b64_padded - b64_padding;
const std::string_view b64_padding_string = b64_padding == 2 ? "=="sv : b64_padding == 1 ? "="sv : ""sv;
if (bytes.size() == b64_unpadded ||
(b64_padding > 0 && bytes.size() == b64_padded && bytes.substr(b64_unpadded) == b64_padding_string)) {
const std::string_view b64_padding_string = b64_padding == 2 ? "=="sv
: b64_padding == 1 ? "="sv
: ""sv;
if (bytes.size() == b64_unpadded || (b64_padding > 0 && bytes.size() == b64_padded &&
bytes.substr(b64_unpadded) == b64_padding_string)) {
if (oxenc::is_base64(bytes)) {
oxenc::from_base64(bytes.begin(), bytes.end(), val_data);
return;
@ -39,4 +43,4 @@ namespace tools {
throw std::runtime_error{"Internal error: invalid binary encoding"};
}
}
} // namespace tools

56
src/common/json_binary_proxy.h
View file

@ -1,11 +1,12 @@
#pragma once
#include "ringct/rctTypes.h"
#include "crypto/crypto.h"
#include <string_view>
#include <nlohmann/json.hpp>
#include <string_view>
#include <unordered_set>
#include "crypto/crypto.h"
#include "ringct/rctTypes.h"
using namespace std::literals;
namespace tools {
@ -14,12 +15,18 @@ namespace tools {
// base64; for bt-encoded requests these can be accepted as binary, hex, or base64.
template <typename T>
inline constexpr bool json_is_binary = false;
template <> inline constexpr bool json_is_binary<crypto::hash> = true;
template <> inline constexpr bool json_is_binary<crypto::public_key> = true;
template <> inline constexpr bool json_is_binary<crypto::ed25519_public_key> = true;
template <> inline constexpr bool json_is_binary<crypto::x25519_public_key> = true;
template <> inline constexpr bool json_is_binary<crypto::key_image> = true;
template <> inline constexpr bool json_is_binary<rct::key> = true;
template <>
inline constexpr bool json_is_binary<crypto::hash> = true;
template <>
inline constexpr bool json_is_binary<crypto::public_key> = true;
template <>
inline constexpr bool json_is_binary<crypto::ed25519_public_key> = true;
template <>
inline constexpr bool json_is_binary<crypto::x25519_public_key> = true;
template <>
inline constexpr bool json_is_binary<crypto::key_image> = true;
template <>
inline constexpr bool json_is_binary<rct::key> = true;
template <typename T>
inline constexpr bool json_is_binary_container = false;
@ -29,13 +36,17 @@ namespace tools {
inline constexpr bool json_is_binary_container<std::unordered_set<T>> = json_is_binary<T>;
// De-referencing wrappers around the above:
template <typename T> inline constexpr bool json_is_binary<const T&> = json_is_binary<T>;
template <typename T> inline constexpr bool json_is_binary<T&&> = json_is_binary<T>;
template <typename T> inline constexpr bool json_is_binary_container<const T&> = json_is_binary_container<T>;
template <typename T> inline constexpr bool json_is_binary_container<T&&> = json_is_binary_container<T>;
template <typename T>
inline constexpr bool json_is_binary<const T&> = json_is_binary<T>;
template <typename T>
inline constexpr bool json_is_binary<T&&> = json_is_binary<T>;
template <typename T>
inline constexpr bool json_is_binary_container<const T&> = json_is_binary_container<T>;
template <typename T>
inline constexpr bool json_is_binary_container<T&&> = json_is_binary_container<T>;
void load_binary_parameter_impl(std::string_view bytes, size_t raw_size, bool allow_raw, uint8_t* val_data);
void load_binary_parameter_impl(
std::string_view bytes, size_t raw_size, bool allow_raw, uint8_t* val_data);
// Loads a binary value from a string_view which may contain hex, base64, and (optionally) raw
// bytes.
@ -50,8 +61,7 @@ namespace tools {
public:
nlohmann::json& e;
enum class fmt { bt, hex, base64 } format;
explicit json_binary_proxy(nlohmann::json& elem, fmt format)
: e{elem}, format{format} {}
explicit json_binary_proxy(nlohmann::json& elem, fmt format) : e{elem}, format{format} {}
json_binary_proxy() = delete;
json_binary_proxy(const json_binary_proxy&) = default;
@ -76,9 +86,12 @@ namespace tools {
/// Assigns binary data from a string_view over a 1-byte, non-char type (e.g. unsigned char or
/// uint8_t).
template <typename Char, std::enable_if_t<sizeof(Char) == 1 && !std::is_same_v<Char, char>, int> = 0>
template <
typename Char,
std::enable_if_t<sizeof(Char) == 1 && !std::is_same_v<Char, char>, int> = 0>
nlohmann::json& operator=(std::basic_string_view<Char> binary_data) {
return *this = std::string_view{reinterpret_cast<const char*>(binary_data.data()), binary_data.size()};
return *this = std::string_view{
reinterpret_cast<const char*>(binary_data.data()), binary_data.size()};
}
/// Takes a trivial, no-padding data structure (e.g. a crypto::hash) as the value and dumps its
@ -117,7 +130,7 @@ namespace tools {
}
};
}
} // namespace tools
// Specializations of binary types for deserialization; when receiving these from json we expect
// them encoded in hex or base64. These may *not* be used for serialization, and will throw if so
@ -134,5 +147,4 @@ namespace nlohmann {
tools::load_binary_parameter(j.get<std::string_view>(), false /*no raw*/, val);
}
};
}
} // namespace nlohmann

5
src/common/lock.h
View file

@ -36,11 +36,10 @@ namespace tools {
/// Takes any number of lockable objects, locks them atomically, and returns a tuple of
/// std::unique_lock holding the individual locks.
template <typename... T>
[[nodiscard]]
std::tuple<std::unique_lock<T>...> unique_locks(T& ...lockables) {
[[nodiscard]] std::tuple<std::unique_lock<T>...> unique_locks(T&... lockables) {
std::lock(lockables...);
auto locks = std::make_tuple(std::unique_lock<T>(lockables, std::adopt_lock)...);
return locks;
}
}
} // namespace tools

2
src/common/median.h
View file

@ -30,4 +30,4 @@ namespace tools {
T median(std::vector<T> v, bool first = false) {
return median(v.begin(), v.end(), first);
}
}
} // namespace tools

20
src/common/meta.h
View file

@ -1,10 +1,12 @@
#pragma once
#include <string>
#include <oxenc/variant.h>
#include <array>
#include <string>
#include <typeinfo>
#ifdef __GNUG__
#include <cxxabi.h>
#include <cstdlib>
#endif
@ -14,22 +16,26 @@ namespace detail {
template <typename T, typename T1, typename... Ts>
constexpr size_t template_index_impl_inner() {
if constexpr (std::is_same_v<T, T1>) return 0;
if constexpr (std::is_same_v<T, T1>)
return 0;
else {
static_assert(sizeof...(Ts) > 0, "Type not found");
return 1 + template_index_impl_inner<T, Ts...>();
}
}
template <typename T, typename C> struct template_index_impl {};
template <typename T, typename C>
struct template_index_impl {};
template <typename T, template <typename...> typename C, typename... Ts>
struct template_index_impl<T, C<Ts...>> : std::integral_constant<size_t, template_index_impl_inner<T, Ts...>()> {};
struct template_index_impl<T, C<Ts...>>
: std::integral_constant<size_t, template_index_impl_inner<T, Ts...>()> {};
} // namespace detail
/// Type wrapper that contains an arbitrary list of types.
template <typename...> struct type_list {};
template <typename...>
struct type_list {};
/// Accesses the index of the first T within a template type's type list. E.g.
///
@ -70,6 +76,8 @@ inline std::string type_name(const std::type_info& ti) {
/// Same as above, but uses a templated type instead of a type_info argument.
template <typename T>
inline std::string type_name() { return type_name(typeid(T)); }
inline std::string type_name() {
return type_name(typeid(T));
}
} // namespace tools

17
src/common/notify.cpp
View file

@ -26,14 +26,14 @@
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "string_util.h"
#include "notify.h"
#include "epee/misc_log_ex.h"
#include "logging/oxen_logger.h"
#include "spawn.h"
#include "notify.h"
#include "string_util.h"
namespace tools
{
namespace tools {
static auto logcat = log::Cat("notify");
@ -42,8 +42,7 @@ namespace tools
- Improve tokenization to handle paths containing whitespaces, quotes, etc.
- Windows unicode support (implies implementing unicode command line parsing code)
*/
Notify::Notify(std::string_view spec)
{
Notify::Notify(std::string_view spec) {
CHECK_AND_ASSERT_THROW_MES(!spec.empty(), "Empty spec");
auto pieces = tools::split_any(spec, " \t", true);
@ -56,8 +55,8 @@ Notify::Notify(std::string_view spec)
args.emplace_back(piece);
}
void Notify::replace_tag(std::vector<std::string>& margs, std::string_view tag, std::string_view value)
{
void Notify::replace_tag(
std::vector<std::string>& margs, std::string_view tag, std::string_view value) {
if (tag.empty())
return;
// Skip margs[0], it's the binary name
@ -74,4 +73,4 @@ int Notify::spawn(const std::vector<std::string>& margs) const {
return tools::spawn(filename, margs, false);
}
}
} // namespace tools

21
src/common/notify.h
View file

@ -28,18 +28,18 @@
#pragma once
#include <fmt/core.h>
#include <string>
#include <string_view>
#include <vector>
#include <fmt/core.h>
#include "common/format.h"
#include "fs.h"
namespace tools
{
namespace tools {
class Notify
{
class Notify {
public:
explicit Notify(std::string_view spec);
@ -57,13 +57,18 @@ private:
int spawn(const std::vector<std::string>& margs) const;
template <typename T, typename... MoreTags>
static void replace_tags(std::vector<std::string>& margs, std::string_view tag, const T& value, MoreTags&&... more) {
static void replace_tags(
std::vector<std::string>& margs,
std::string_view tag,
const T& value,
MoreTags&&... more) {
replace_tag(margs, tag, "{}"_format(value));
if constexpr (sizeof...(MoreTags) > 0)
replace_tags(margs, std::forward<MoreTags>(more)...);
}
static void replace_tag(std::vector<std::string>& margs, std::string_view tag, std::string_view value);
static void replace_tag(
std::vector<std::string>& margs, std::string_view tag, std::string_view value);
};
}
} // namespace tools

51
src/common/oxen.cpp
View file

@ -18,14 +18,15 @@
/* Specification. */
#include <limits>
#include <cfloat>
#include <cmath>
#include <limits>
// TODO(oxen): This is temporary until we switch to integer math for calculating
// block rewards. We provide the specific implementation to minimise the risk of
// different results from math functions across different std libraries.
static_assert(std::numeric_limits<double>::is_iec559, "We require IEEE standard compliant doubles.");
static_assert(
std::numeric_limits<double>::is_iec559, "We require IEEE standard compliant doubles.");
/* Best possible approximation of log(2) as a 'double'. */
#define LOG2 0.693147180559945309417232121458176568075
@ -39,9 +40,7 @@ static_assert(std::numeric_limits<double>::is_iec559, "We require IEEE standard
/* Best possible approximation of 256/log(2) as a 'double'. */
#define LOG2_BY_256_INVERSE 369.329930467574632284140718336484387181
double
oxen::exp2(double x)
{
double oxen::exp2(double x) {
/* exp2(x) = exp(x*log(2)).
If we would compute it like this, there would be rounding errors for
integer or near-integer values of x. To avoid these, we inline the
@ -108,11 +107,7 @@ oxen::exp2(double x)
#define TANH_COEFF_15 -0.00145583438705131826824948518070211191904
double z2 = z * z;
double tanh_z =
((TANH_COEFF_5
* z2 + TANH_COEFF_3)
* z2 + TANH_COEFF_1)
* z;
double tanh_z = ((TANH_COEFF_5 * z2 + TANH_COEFF_3) * z2 + TANH_COEFF_1) * z;
double exp_y = (1.0 + tanh_z) / (1.0 - tanh_z);
@ -127,8 +122,7 @@ oxen::exp2(double x)
(dotimes (i 257)
(format t " ~D,~%"
(float (exp (* (/ (- i 128) 256) (log 2L0))) a))) */
static const double exp_table[257] =
{
static const double exp_table[257] = {
0.707106781186547524400844362104849039284,
0.709023942160207598920563322257676190836,
0.710946301084582779904674297352120049962,
@ -385,8 +379,7 @@ oxen::exp2(double x)
1.40277269122020470637471352433337881711,
1.40657599381901544248361973255451684411,
1.410389608217270704414375128268675481145,
1.41421356237309504880168872420969807857
};
1.41421356237309504880168872420969807857};
double ret = exp_table[128 + m] * exp_y;
for (int i = 0; i < n; i++)
@ -416,8 +409,8 @@ oxen::exp2(double x)
/* Specification. */
#include <cstdint>
#include <cfloat>
#include <cstdint>
#include <limits>
/* -0.0. See minus-zero.h. */
@ -433,19 +426,17 @@ oxen::exp2(double x)
#pragma fenv_access(off)
#endif
double
oxen::round (double x)
{
double oxen::round(double x) {
/* 2^(DBL_MANT_DIG-1). */
static const double TWO_MANT_DIG =
/* Assume DBL_MANT_DIG <= 5 * 31.
Use the identity
n = floor(n/5) + floor((n+1)/5) + ... + floor((n+4)/5). */
(double) (1U << ((DBL_MANT_DIG - 1) / 5))
* (double) (1U << ((DBL_MANT_DIG - 1 + 1) / 5))
* (double) (1U << ((DBL_MANT_DIG - 1 + 2) / 5))
* (double) (1U << ((DBL_MANT_DIG - 1 + 3) / 5))
* (double) (1U << ((DBL_MANT_DIG - 1 + 4) / 5));
(double)(1U << ((DBL_MANT_DIG - 1) / 5)) *
(double)(1U << ((DBL_MANT_DIG - 1 + 1) / 5)) *
(double)(1U << ((DBL_MANT_DIG - 1 + 2) / 5)) *
(double)(1U << ((DBL_MANT_DIG - 1 + 3) / 5)) *
(double)(1U << ((DBL_MANT_DIG - 1 + 4) / 5));
/* The use of 'volatile' guarantees that excess precision bits are dropped at
each addition step and before the following comparison at the caller's
@ -456,14 +447,12 @@ oxen::round (double x)
volatile double y = x;
volatile double z = y;
if (z > 0.0)
{
if (z > 0.0) {
/* Avoid rounding error for x = 0.5 - 2^(-DBL_MANT_DIG-1). */
if (z < 0.5)
z = 0.0;
/* Avoid rounding errors for values near 2^k, where k >= DBL_MANT_DIG-1. */
else if (z < TWO_MANT_DIG)
{
else if (z < TWO_MANT_DIG) {
/* Add 0.5 to the absolute value. */
y = z += 0.5;
/* Round to the next integer (nearest or up or down, doesn't
@ -474,15 +463,12 @@ oxen::round (double x)
if (z > y)
z -= 1.0;
}
}
else if (z < 0.0)
{
} else if (z < 0.0) {
/* Avoid rounding error for x = -(0.5 - 2^(-DBL_MANT_DIG-1)). */
if (z > -0.5)
z = MINUS_ZERO;
/* Avoid rounding errors for values near -2^k, where k >= DBL_MANT_DIG-1. */
else if (z > -TWO_MANT_DIG)
{
else if (z > -TWO_MANT_DIG) {
/* Add 0.5 to the absolute value. */
y = z -= 0.5;
/* Round to the next integer (nearest or up or down, doesn't
@ -496,4 +482,3 @@ oxen::round (double x)
}
return z;
}

53
src/common/oxen.h
View file

@ -33,49 +33,68 @@
#include <utility>
#define OXEN_RPC_DOC_INTROSPECT
namespace oxen
{
namespace oxen {
double round(double);
double exp2(double);
template <typename lambda_t>
struct deferred
{
struct deferred {
private:
lambda_t lambda;
bool cancelled = false;
public:
deferred(lambda_t lambda) : lambda(lambda) {}
void invoke() { lambda(); cancelled = true; } // Invoke early instead of at destruction
void invoke() {
lambda();
cancelled = true;
} // Invoke early instead of at destruction
void cancel() { cancelled = true; } // Cancel invocation at destruction
~deferred() { if (!cancelled) lambda(); }
~deferred() {
if (!cancelled)
lambda();
}
deferred(deferred<lambda_t>&& d) : lambda{std::move(d.lambda)}, cancelled{d.cancelled} { d.cancel(); }
deferred& operator=(deferred<lambda_t>&& d) { lambda = std::move(d.lambda); cancelled = d.cancelled; d.cancel(); return *this; }
deferred(deferred<lambda_t>&& d) : lambda{std::move(d.lambda)}, cancelled{d.cancelled} {
d.cancel();
}
deferred& operator=(deferred<lambda_t>&& d) {
lambda = std::move(d.lambda);
cancelled = d.cancelled;
d.cancel();
return *this;
}
deferred(const deferred<lambda_t>&) = delete;
deferred& operator=(const deferred<lambda_t>&) = delete;
};
template <typename lambda_t>
[[nodiscard]]
deferred<lambda_t> defer(lambda_t lambda) { return lambda; }
[[nodiscard]] deferred<lambda_t> defer(lambda_t lambda) {
return lambda;
}
struct defer_helper
{
struct defer_helper {
template <typename lambda_t>
deferred<lambda_t> operator+(lambda_t lambda) { return lambda; }
deferred<lambda_t> operator+(lambda_t lambda) {
return lambda;
}
};
#define OXEN_TOKEN_COMBINE2(x, y) x##y
#define OXEN_TOKEN_COMBINE(x, y) OXEN_TOKEN_COMBINE2(x, y)
#define OXEN_DEFER auto const OXEN_TOKEN_COMBINE(oxen_defer_, __LINE__) = oxen::defer_helper() + [&]()
#define OXEN_DEFER \
auto const OXEN_TOKEN_COMBINE(oxen_defer_, __LINE__) = oxen::defer_helper() + [&]()
template <typename T, size_t N>
constexpr size_t array_count(T (&)[N]) { return N; }
constexpr size_t array_count(T (&)[N]) {
return N;
}
template <typename T, size_t N>
constexpr size_t char_count(T (&)[N]) { return N - 1; }
constexpr size_t char_count(T (&)[N]) {
return N - 1;
}
}; // namespace Oxen
}; // namespace oxen
#endif // OXEN_H

140
src/common/password.cpp
View file

@ -30,10 +30,10 @@
#include "password.h"
#include <iostream>
#include <cstdio>
#include <utility>
#include <iostream>
#include <type_traits>
#include <utility>
#if defined(_WIN32)
#include <io.h>
@ -45,16 +45,13 @@
#define EOT 0x4
namespace
{
namespace {
#if defined(_WIN32)
bool is_cin_tty() noexcept
{
bool is_cin_tty() noexcept {
return 0 != _isatty(_fileno(stdin));
}
bool read_from_tty(epee::wipeable_string& pass, bool hide_input)
{
bool read_from_tty(epee::wipeable_string& pass, bool hide_input) {
HANDLE h_cin = ::GetStdHandle(STD_INPUT_HANDLE);
DWORD mode_old;
@ -66,26 +63,19 @@ namespace
pass.reserve(tools::password_container::max_password_size);
std::vector<int> chlen;
chlen.reserve(tools::password_container::max_password_size);
while (pass.size() < tools::password_container::max_password_size)
{
while (pass.size() < tools::password_container::max_password_size) {
DWORD read;
wchar_t ucs2_ch;
r = (TRUE == ::ReadConsoleW(h_cin, &ucs2_ch, 1, &read, NULL));
r &= (1 == read);
if (!r)
{
if (!r) {
break;
}
else if (ucs2_ch == L'\r')
{
} else if (ucs2_ch == L'\r') {
std::cout << std::endl;
break;
}
else if (ucs2_ch == L'\b')
{
if (!pass.empty())
{
} else if (ucs2_ch == L'\b') {
if (!pass.empty()) {
int len = chlen.back();
chlen.pop_back();
while (len-- > 0)
@ -96,7 +86,8 @@ namespace
char utf8_ch[8] = {0};
int len;
if((len = WideCharToMultiByte(CP_UTF8, 0, &ucs2_ch, 1, utf8_ch, sizeof(utf8_ch), NULL, NULL)) <= 0)
if ((len = WideCharToMultiByte(
CP_UTF8, 0, &ucs2_ch, 1, utf8_ch, sizeof(utf8_ch), NULL, NULL)) <= 0)
break;
if (pass.size() + len >= tools::password_container::max_password_size)
@ -113,13 +104,11 @@ namespace
#else // end WIN32
bool is_cin_tty() noexcept
{
bool is_cin_tty() noexcept {
return 0 != isatty(fileno(stdin));
}
int getch(bool hide_input) noexcept
{
int getch(bool hide_input) noexcept {
struct termios tty_old;
tcgetattr(STDIN_FILENO, &tty_old);
@ -135,39 +124,27 @@ namespace
return ch;
}
bool read_from_tty(epee::wipeable_string& aPass, bool hide_input)
{
bool read_from_tty(epee::wipeable_string& aPass, bool hide_input) {
static constexpr const char BACKSPACE = 127;
aPass.reserve(tools::password_container::max_password_size);
while (aPass.size() < tools::password_container::max_password_size)
{
while (aPass.size() < tools::password_container::max_password_size) {
int ch = getch(hide_input);
if (EOF == ch || ch == EOT)
{
if (EOF == ch || ch == EOT) {
return false;
}
else if (ch == '\n' || ch == '\r')
{
} else if (ch == '\n' || ch == '\r') {
std::cout << std::endl;
break;
}
else if (ch == BACKSPACE)
{
if (!aPass.empty())
{
} else if (ch == BACKSPACE) {
if (!aPass.empty()) {
aPass.pop_back();
if (!hide_input)
std::cout << "\b\b\b \b\b\b" << std::flush;
}
else
{
} else {
if (!hide_input)
std::cout << "\b\b \b\b" << std::flush;
}
}
else
{
} else {
aPass.push_back(ch);
}
}
@ -177,29 +154,28 @@ namespace
#endif // end !WIN32
bool read_from_tty(const bool verify, const char *message, bool hide_input, epee::wipeable_string& pass1, epee::wipeable_string& pass2)
{
while (true)
{
bool read_from_tty(
const bool verify,
const char* message,
bool hide_input,
epee::wipeable_string& pass1,
epee::wipeable_string& pass2) {
while (true) {
if (message)
std::cout << message << ": " << std::flush;
if (!read_from_tty(pass1, hide_input))
return false;
if (verify)
{
if (verify) {
std::cout << "Confirm password: ";
if (!read_from_tty(pass2, hide_input))
return false;
if(pass1!=pass2)
{
if (pass1 != pass2) {
std::cout << "Passwords do not match! Please try again." << std::endl;
pass1.clear();
pass2.clear();
}
else //new password matches
} else // new password matches
return true;
}
else
} else
return true;
// No need to verify password entered at this point in the code
}
@ -207,22 +183,15 @@ namespace
return false;
}
bool read_from_file(epee::wipeable_string& pass)
{
bool read_from_file(epee::wipeable_string& pass) {
pass.reserve(tools::password_container::max_password_size);
for (size_t i = 0; i < tools::password_container::max_password_size; ++i)
{
for (size_t i = 0; i < tools::password_container::max_password_size; ++i) {
char ch = static_cast<char>(std::cin.get());
if (std::cin.eof() || ch == '\n' || ch == '\r')
{
if (std::cin.eof() || ch == '\n' || ch == '\r') {
break;
}
else if (std::cin.fail())
{
} else if (std::cin.fail()) {
return false;
}
else
{
} else {
pass.push_back(ch);
}
}
@ -231,24 +200,22 @@ namespace
} // anonymous namespace
namespace tools
{
namespace tools {
// deleted via private member
password_container::password_container() noexcept : m_password() {}
password_container::password_container(epee::wipeable_string password) noexcept
: m_password{std::move(password)}
{
}
password_container::password_container(epee::wipeable_string password) noexcept :
m_password{std::move(password)} {}
std::atomic<bool> password_container::is_prompting(false);
std::optional<password_container> password_container::prompt(const bool verify, const char *message, bool hide_input)
{
std::optional<password_container> password_container::prompt(
const bool verify, const char* message, bool hide_input) {
is_prompting = true;
password_container pass1{};
password_container pass2{};
if (is_cin_tty() ? read_from_tty(verify, message, hide_input, pass1.m_password, pass2.m_password) : read_from_file(pass1.m_password))
{
if (is_cin_tty()
? read_from_tty(verify, message, hide_input, pass1.m_password, pass2.m_password)
: read_from_file(pass1.m_password)) {
is_prompting = false;
return {std::move(pass1)};
}
@ -257,21 +224,20 @@ namespace tools
return std::nullopt;
}
std::optional<login> login::parse(std::string&& userpass, bool verify, const std::function<std::optional<password_container>(bool)> &prompt)
{
std::optional<login> login::parse(
std::string&& userpass,
bool verify,
const std::function<std::optional<password_container>(bool)>& prompt) {
login out{};
const auto loc = userpass.find(':');
if (loc == std::string::npos)
{
if (loc == std::string::npos) {
auto result = prompt(verify);
if (!result)
return std::nullopt;
out.password = std::move(*result);
}
else
{
} else {
out.password = password_container{userpass.substr(loc + 1)};
}
@ -279,4 +245,4 @@ namespace tools
password_container wipe{std::move(userpass)};
return {std::move(out)};
}
}
} // namespace tools

30
src/common/password.h
View file

@ -30,17 +30,16 @@
#pragma once
#include <string>
#include <atomic>
#include <optional>
#include <functional>
#include <cstddef>
#include <functional>
#include <optional>
#include <string>
#include "epee/wipeable_string.h"
namespace tools
{
class password_container
{
namespace tools {
class password_container {
public:
static constexpr const size_t max_password_size = 1024;
@ -51,7 +50,8 @@ namespace tools
password_container(epee::wipeable_string password) noexcept;
//! \return A password from stdin TTY prompt or `std::cin` pipe.
static std::optional<password_container> prompt(bool verify, const char *mesage = "Password", bool hide_input = true);
static std::optional<password_container> prompt(
bool verify, const char* mesage = "Password", bool hide_input = true);
static std::atomic<bool> is_prompting;
const epee::wipeable_string& password() const noexcept { return m_password; }
@ -60,13 +60,12 @@ namespace tools
epee::wipeable_string m_password;
};
struct login
{
struct login {
login() = default;
/// Constructs a login from a username/password. Does not prompt.
login(std::string_view user, epee::wipeable_string pass)
: username{user}, password{std::move(pass)} {}
login(std::string_view user, epee::wipeable_string pass) :
username{user}, password{std::move(pass)} {}
/*!
Extracts username and password from the format `username:password`. A
@ -81,9 +80,12 @@ namespace tools
\return The username and password, or std::nullopt if
`password_container::prompt` fails.
*/
static std::optional<login> parse(std::string&& userpass, bool verify, const std::function<std::optional<password_container>(bool)> &prompt);
static std::optional<login> parse(
std::string&& userpass,
bool verify,
const std::function<std::optional<password_container>(bool)>& prompt);
std::string username;
password_container password;
};
}
} // namespace tools

38
src/common/periodic_task.h
View file

@ -1,36 +1,36 @@
#pragma once
#include <chrono>
#include <atomic>
#include <chrono>
#include "crypto/crypto.h"
namespace tools
{
namespace tools {
// Periodic timer that gatekeeps calling of a job to a minimum interval after the previous job
// finished. Only the reset() call is thread-safe; everything else should be confined to the
// owning thread.
class periodic_task
{
class periodic_task {
public:
explicit periodic_task(std::chrono::microseconds interval,
explicit periodic_task(
std::chrono::microseconds interval,
bool start_immediately = true,
std::pair<int, int> random_delay_interval = {})
: m_interval{interval}
, m_last_worked_time{std::chrono::steady_clock::now()}
, m_trigger_now{start_immediately}
, m_random_delay_interval{random_delay_interval}
, m_next_delay{std::chrono::microseconds(crypto::rand_range(m_random_delay_interval.first, m_random_delay_interval.second))}
{}
std::pair<int, int> random_delay_interval = {}) :
m_interval{interval},
m_last_worked_time{std::chrono::steady_clock::now()},
m_trigger_now{start_immediately},
m_random_delay_interval{random_delay_interval},
m_next_delay{std::chrono::microseconds(crypto::rand_range(
m_random_delay_interval.first, m_random_delay_interval.second))} {}
template <class functor_t>
void do_call(functor_t functr)
{
if (m_trigger_now || std::chrono::steady_clock::now() - m_last_worked_time > (m_interval + m_next_delay))
{
void do_call(functor_t functr) {
if (m_trigger_now ||
std::chrono::steady_clock::now() - m_last_worked_time > (m_interval + m_next_delay)) {
functr();
m_last_worked_time = std::chrono::steady_clock::now();
m_trigger_now = false;
m_next_delay = std::chrono::microseconds(crypto::rand_range(m_random_delay_interval.first, m_random_delay_interval.second));
m_next_delay = std::chrono::microseconds(crypto::rand_range(
m_random_delay_interval.first, m_random_delay_interval.second));
}
}
@ -48,4 +48,4 @@ private:
std::pair<int, int> m_random_delay_interval;
std::chrono::microseconds m_next_delay;
};
};
}; // namespace tools

55
src/common/pruning.cpp
View file

@ -26,25 +26,25 @@
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "cryptonote_config.h"
#include "epee/misc_log_ex.h"
#include "crypto/crypto.h"
#include "pruning.h"
namespace tools
{
#include "crypto/crypto.h"
#include "cryptonote_config.h"
#include "epee/misc_log_ex.h"
namespace tools {
using namespace cryptonote;
uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes)
{
CHECK_AND_ASSERT_THROW_MES(log_stripes <= PRUNING_SEED_LOG_STRIPES_MASK, "log_stripes out of range");
uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes) {
CHECK_AND_ASSERT_THROW_MES(
log_stripes <= PRUNING_SEED_LOG_STRIPES_MASK, "log_stripes out of range");
CHECK_AND_ASSERT_THROW_MES(stripe > 0 && stripe <= (1ul << log_stripes), "stripe out of range");
return (log_stripes << PRUNING_SEED_LOG_STRIPES_SHIFT) | ((stripe - 1) << PRUNING_SEED_STRIPE_SHIFT);
return (log_stripes << PRUNING_SEED_LOG_STRIPES_SHIFT) |
((stripe - 1) << PRUNING_SEED_STRIPE_SHIFT);
}
bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed)
{
bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) {
const uint32_t stripe = get_pruning_stripe(pruning_seed);
if (stripe == 0)
return true;
@ -53,25 +53,26 @@ bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint3
return block_stripe == 0 || block_stripe == stripe;
}
uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes)
{
uint32_t get_pruning_stripe(
uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes) {
if (block_height + PRUNING_TIP_BLOCKS >= blockchain_height)
return 0;
return ((block_height / PRUNING_STRIPE_SIZE) & (uint64_t)((1ul << log_stripes) - 1)) + 1;
}
uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes)
{
uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes) {
const uint32_t stripe = get_pruning_stripe(block_height, blockchain_height, log_stripes);
if (stripe == 0)
return 0;
return make_pruning_seed(stripe, log_stripes);
}
uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed)
{
CHECK_AND_ASSERT_MES(block_height <= MAX_BLOCK_NUMBER+1, block_height, "block_height too large");
CHECK_AND_ASSERT_MES(blockchain_height <= MAX_BLOCK_NUMBER+1, block_height, "blockchain_height too large");
uint64_t get_next_unpruned_block_height(
uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) {
CHECK_AND_ASSERT_MES(
block_height <= MAX_BLOCK_NUMBER + 1, block_height, "block_height too large");
CHECK_AND_ASSERT_MES(
blockchain_height <= MAX_BLOCK_NUMBER + 1, block_height, "blockchain_height too large");
const uint32_t stripe = get_pruning_stripe(pruning_seed);
if (stripe == 0)
return block_height;
@ -85,15 +86,16 @@ uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockcha
return block_height;
const uint64_t cycles = ((block_height / PRUNING_STRIPE_SIZE) >> log_stripes);
const uint64_t cycle_start = cycles + ((stripe > block_pruning_stripe) ? 0 : 1);
const uint64_t h = cycle_start * (PRUNING_STRIPE_SIZE << log_stripes) + (stripe - 1) * PRUNING_STRIPE_SIZE;
const uint64_t h =
cycle_start * (PRUNING_STRIPE_SIZE << log_stripes) + (stripe - 1) * PRUNING_STRIPE_SIZE;
if (h + PRUNING_TIP_BLOCKS > blockchain_height)
return blockchain_height < PRUNING_TIP_BLOCKS ? 0 : blockchain_height - PRUNING_TIP_BLOCKS;
CHECK_AND_ASSERT_MES(h >= block_height, block_height, "h < block_height, unexpected");
return h;
}
uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed)
{
uint64_t get_next_pruned_block_height(
uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) {
const uint32_t stripe = get_pruning_stripe(pruning_seed);
if (stripe == 0)
return blockchain_height;
@ -106,13 +108,12 @@ uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain
if (block_pruning_seed != stripe)
return block_height;
const uint32_t next_stripe = 1 + (block_pruning_seed & mask);
return get_next_unpruned_block_height(block_height, blockchain_height, tools::make_pruning_seed(next_stripe, log_stripes));
return get_next_unpruned_block_height(
block_height, blockchain_height, tools::make_pruning_seed(next_stripe, log_stripes));
}
uint32_t get_random_stripe()
{
uint32_t get_random_stripe() {
return 1 + crypto::rand<uint8_t>() % (1ul << PRUNING_LOG_STRIPES);
}
}
} // namespace tools

15
src/common/pruning.h
View file

@ -30,8 +30,7 @@
#include <stdint.h>
namespace tools
{
namespace tools {
inline constexpr uint32_t PRUNING_SEED_LOG_STRIPES_SHIFT = 7;
inline constexpr uint32_t PRUNING_SEED_LOG_STRIPES_MASK = 0x7;
inline constexpr uint32_t PRUNING_SEED_STRIPE_SHIFT = 0;
@ -49,10 +48,12 @@ namespace tools
uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes);
bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed);
uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes);
uint32_t get_pruning_stripe(
uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes);
uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes);
uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed);
uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed);
uint64_t get_next_unpruned_block_height(
uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed);
uint64_t get_next_pruned_block_height(
uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed);
uint32_t get_random_stripe();
}
} // namespace tools

10
src/common/random.cpp
View file

@ -27,19 +27,21 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "random.h"
#include <cassert>
namespace tools {
thread_local std::mt19937_64 rng{std::random_device{}()};
uint64_t uniform_distribution_portable(std::mt19937_64& rng, const uint64_t n)
{
uint64_t uniform_distribution_portable(std::mt19937_64& rng, const uint64_t n) {
assert(n > 0);
const uint64_t secureMax = rng.max() - rng.max() % n;
uint64_t x;
do x = rng(); while (x >= secureMax);
do
x = rng();
while (x >= secureMax);
return x / (secureMax / n);
}
}
} // namespace tools

20
src/common/random.h
View file

@ -44,31 +44,33 @@ uint64_t uniform_distribution_portable(std::mt19937_64& rng, uint64_t n);
/// Uniformly shuffles all the elements in [begin, end) in a deterministic method so that, given the
/// same seed, this will always produce the same result on any platform/compiler/etc.
template <typename RandomIt>
void shuffle_portable(RandomIt begin, RandomIt end, std::mt19937_64 &rng)
{
if (end <= begin + 1) return;
void shuffle_portable(RandomIt begin, RandomIt end, std::mt19937_64& rng) {
if (end <= begin + 1)
return;
const size_t size = std::distance(begin, end);
for (size_t i = 1; i < size; i++)
{
for (size_t i = 1; i < size; i++) {
size_t j = (size_t)uniform_distribution_portable(rng, i + 1);
using std::swap;
swap(begin[i], begin[j]);
}
}
/// Returns a random element between the elements in [begin, end) will use the random number generator provided as g
/// Returns a random element between the elements in [begin, end) will use the random number
/// generator provided as g
template <typename Iter, typename RandomGenerator>
Iter select_randomly(Iter begin, Iter end, RandomGenerator& g) {
auto dist = std::distance(begin, end);
if (dist <= 1) return begin; // Handles both begin=end case and single-element case
if (dist <= 1)
return begin; // Handles both begin=end case and single-element case
std::advance(begin, std::uniform_int_distribution<>{0, static_cast<int>(dist - 1)}(g));
return begin;
}
/// Returns a random element same as above but defaults to the seeded random number generator defined in this file
/// Returns a random element same as above but defaults to the seeded random number generator
/// defined in this file
template <typename Iter>
Iter select_randomly(Iter begin, Iter end) {
return select_randomly(begin, end, rng);
}
};
}; // namespace tools

29
src/common/rules.cpp
View file

@ -28,33 +28,28 @@
//
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
#include <ctime>
#include <cstdint>
#include "cryptonote_config.h"
#include <ctime>
#include "common/util.h"
#include "cryptonote_config.h"
namespace cryptonote
{
namespace cryptonote { namespace rules {
namespace rules
{
bool is_output_unlocked(uint64_t unlock_time, uint64_t height)
{
if(unlock_time < MAX_BLOCK_NUMBER)
{
bool is_output_unlocked(uint64_t unlock_time, uint64_t height) {
if (unlock_time < MAX_BLOCK_NUMBER) {
// ND: Instead of calling get_current_blockchain_height(), call m_db->height()
// directly as get_current_blockchain_height() locks the recursive mutex.
if (height - 1 + LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time)
return true;
else
return false;
}
else
{
} else {
// interpret as time
uint64_t current_time = static_cast<uint64_t>(time(NULL));
if(current_time + tools::to_seconds(LOCKED_TX_ALLOWED_DELTA_BLOCKS * TARGET_BLOCK_TIME) >= unlock_time)
if (current_time +
tools::to_seconds(LOCKED_TX_ALLOWED_DELTA_BLOCKS * TARGET_BLOCK_TIME) >=
unlock_time)
return true;
else
return false;
@ -62,6 +57,4 @@ bool is_output_unlocked(uint64_t unlock_time, uint64_t height)
return false;
}
} // namespace rules
} // namespace cryptonote
}} // namespace cryptonote::rules

3
src/common/rules.h
View file

@ -30,8 +30,7 @@
#include <cstdint>
namespace cryptonote::rules
{
namespace cryptonote::rules {
bool is_output_unlocked(uint64_t unlock_time, uint64_t height);

15
src/common/scoped_message_writer.cpp
View file

@ -1,30 +1,27 @@
#include "scoped_message_writer.h"
#include "common/format.h"
namespace tools {
static auto logcat = log::Cat("msgwriter");
scoped_message_writer& scoped_message_writer::flush()
{
if (!m_content.empty())
{
scoped_message_writer& scoped_message_writer::flush() {
if (!m_content.empty()) {
logcat->log(m_log_level, "{}{}", m_prefix, m_content);
if (m_color) {
rdln::suspend_readline pause_readline;
fmt::print(fg(*m_color), "{}{}\n", m_prefix, m_content);
}
else
} else
fmt::print("{}{}\n", m_prefix, m_content);
m_content.clear();
}
return *this;
}
scoped_message_writer::~scoped_message_writer()
{
scoped_message_writer::~scoped_message_writer() {
flush();
}
}
} // namespace tools

64
src/common/scoped_message_writer.h
View file

@ -28,40 +28,38 @@
#pragma once
#include "epee/readline_suspend.h"
#include "epee/misc_log_ex.h"
#include <iostream>
#include "logging/oxen_logger.h"
#include <fmt/color.h>
namespace tools
{
#include <iostream>
#include "epee/misc_log_ex.h"
#include "epee/readline_suspend.h"
#include "logging/oxen_logger.h"
namespace tools {
/************************************************************************/
/* */
/************************************************************************/
class scoped_message_writer
{
class scoped_message_writer {
private:
std::string m_prefix;
std::string m_content;
std::optional<fmt::terminal_color> m_color;
oxen::log::Level m_log_level;
public:
explicit scoped_message_writer(
std::optional<fmt::terminal_color> color = std::nullopt,
std::string prefix = "",
log::Level log_level = log::Level::info
)
: m_color{color}, m_log_level{log_level}, m_prefix{std::move(prefix)}
{}
log::Level log_level = log::Level::info) :
m_color{color}, m_log_level{log_level}, m_prefix{std::move(prefix)} {}
scoped_message_writer(scoped_message_writer&& o)
: m_prefix{std::move(o.m_prefix)},
scoped_message_writer(scoped_message_writer&& o) :
m_prefix{std::move(o.m_prefix)},
m_content{std::move(o.m_content)},
m_color{o.m_color},
m_log_level{o.m_log_level}
{
m_log_level{o.m_log_level} {
o.m_content.clear();
}
@ -72,8 +70,7 @@ public:
/// Appends a message and returns *this (so that it can be chained). If called with more than 1
/// argument then the first argument is fmt::format'ed with the remaining arguments.
template <typename... T>
scoped_message_writer& append(std::string_view msg, T&&... args)
{
scoped_message_writer& append(std::string_view msg, T&&... args) {
if constexpr (sizeof...(T))
fmt::format_to(std::back_inserter(m_content), msg, std::forward<T>(args)...);
else
@ -84,10 +81,13 @@ public:
/// Same as .append(msg). (Doesn't format, just like the single-argument .append(msg)).
scoped_message_writer& operator+=(std::string_view msg) { return append(msg); }
/// Essentially the same as +=, but can only be used on an rvalue instance of the object, so that
/// you can do things like: `scoped_message_writer{} + "abc"`, which feels more natural than
/// `scoped_message_writer{} += "abc"`.
scoped_message_writer&& operator+(std::string_view msg) && { append(msg); return std::move(*this); }
/// Essentially the same as +=, but can only be used on an rvalue instance of the object, so
/// that you can do things like: `scoped_message_writer{} + "abc"`, which feels more natural
/// than `scoped_message_writer{} += "abc"`.
scoped_message_writer&& operator+(std::string_view msg) && {
append(msg);
return std::move(*this);
}
/// Flushes the current message to output and resets it. This is normally not called explicitly
/// but rather implicitly when the object is destroyed.
@ -98,8 +98,8 @@ public:
};
template <typename... T>
scoped_message_writer msg_writer(std::optional<fmt::terminal_color> color = std::nullopt, T&&... args)
{
scoped_message_writer msg_writer(
std::optional<fmt::terminal_color> color = std::nullopt, T&&... args) {
scoped_message_writer writer{color};
if constexpr (sizeof...(T))
writer.append(std::forward<T>(args)...);
@ -107,12 +107,10 @@ scoped_message_writer msg_writer(std::optional<fmt::terminal_color> color = std:
}
template <typename... T>
scoped_message_writer msg_writer(std::string_view msg, T&&... args)
{
scoped_message_writer msg_writer(std::string_view msg, T&&... args) {
return msg_writer(std::nullopt, msg, std::forward<T>(args)...);
}
constexpr std::optional<fmt::terminal_color> success_color{fmt::terminal_color::green};
constexpr std::optional<fmt::terminal_color> fail_color{fmt::terminal_color::red};
@ -122,23 +120,20 @@ constexpr std::optional<fmt::terminal_color> fail_color{fmt::terminal_color::red
///
/// (We deduce the Bool argument here to avoid implicit conversion to bool from non-bool values).
template <typename Bool, typename... T, std::enable_if_t<std::is_same_v<Bool, bool>, int> = 0>
scoped_message_writer success_msg_writer(Bool color, T&&... args)
{
scoped_message_writer success_msg_writer(Bool color, T&&... args) {
auto writer = msg_writer(color ? success_color : std::nullopt);
if constexpr (sizeof...(T))
writer.append(std::forward<T>(args)...);
return writer;
}
inline scoped_message_writer success_msg_writer()
{
inline scoped_message_writer success_msg_writer() {
return success_msg_writer(true);
}
/// Same as above, but for calling without just a message (with a bool). Color will be true.
template <typename... T>
scoped_message_writer success_msg_writer(std::string_view msg, T&&... args)
{
scoped_message_writer success_msg_writer(std::string_view msg, T&&... args) {
return success_msg_writer(true, msg, std::forward<T>(args)...);
}
@ -147,8 +142,7 @@ scoped_message_writer success_msg_writer(std::string_view msg, T&&... args)
/// .append() and so can specify either a single unformatted string, or a format string + format
/// arguments.
template <typename... T>
scoped_message_writer fail_msg_writer(T&&... args)
{
scoped_message_writer fail_msg_writer(T&&... args) {
scoped_message_writer writer{fail_color, "Error: ", spdlog::level::err};
if constexpr (sizeof...(T))
writer.append(std::forward<T>(args)...);

20
src/common/sha256sum.cpp
View file

@ -1,23 +1,22 @@
#include "sha256sum.h"
#include <sodium/crypto_hash_sha256.h>
#include <fstream>
#include "crypto/hash.h"
#include "fs.h"
#include <sodium/crypto_hash_sha256.h>
namespace tools {
bool sha256sum_str(std::string_view data, crypto::hash &hash)
{
bool sha256sum_str(std::string_view data, crypto::hash& hash) {
crypto_hash_sha256(
hash.data(),
reinterpret_cast<const unsigned char*>(data.data()),
data.size());
hash.data(), reinterpret_cast<const unsigned char*>(data.data()), data.size());
return true;
}
bool sha256sum_file(const fs::path& filename, crypto::hash& hash)
{
bool sha256sum_file(const fs::path& filename, crypto::hash& hash) {
if (std::error_code ec; !fs::exists(filename, ec) || ec)
return false;
fs::ifstream f;
@ -32,8 +31,7 @@ namespace tools {
f.seekg(0, std::ios::beg);
std::array<unsigned char, 16384> buf;
while (size_left)
{
while (size_left) {
auto read_size = std::min(size_left, buf.size());
f.read(reinterpret_cast<char*>(buf.data()), read_size);
if (!f || !f.good())
@ -46,4 +44,4 @@ namespace tools {
return true;
}
}
} // namespace tools

22
src/common/sha256sum.h
View file

@ -1,10 +1,13 @@
#pragma once
#include <type_traits>
#include <string>
#include <string_view>
#include <type_traits>
#include "fs.h"
namespace crypto { struct hash; }
namespace crypto {
struct hash;
}
namespace tools {
@ -13,21 +16,22 @@ namespace tools {
// Calculates sha256 checksum of the given data, for non-char string_view (e.g.
// basic_string_view<unsigned char> or basic_string_view<uint8_t>).
template <typename Char, std::enable_if_t<sizeof(Char) == 1 && !std::is_same_v<Char, char>, int> = 0>
bool sha256sum_str(std::basic_string_view<Char> str, crypto::hash& hash)
{
return sha256sum_str(std::string_view{reinterpret_cast<const char*>(str.data()), str.size()}, hash);
template <
typename Char,
std::enable_if_t<sizeof(Char) == 1 && !std::is_same_v<Char, char>, int> = 0>
bool sha256sum_str(std::basic_string_view<Char> str, crypto::hash& hash) {
return sha256sum_str(
std::string_view{reinterpret_cast<const char*>(str.data()), str.size()}, hash);
}
// Calculates sha256 checksum of the given byte data given any arbitrary size-1 value pointer and
// byte length.
template <typename Char, std::enable_if_t<sizeof(Char) == 1, int> = 0>
bool sha256sum_str(const Char* data, size_t len, crypto::hash& hash)
{
bool sha256sum_str(const Char* data, size_t len, crypto::hash& hash) {
return sha256sum_str(std::string_view{reinterpret_cast<const char*>(data), len}, hash);
}
// Opens the given file and calculates a sha256sum of its contents
bool sha256sum_file(const fs::path& filename, crypto::hash& hash);
}
} // namespace tools

37
src/common/signal_handler.h
View file

@ -1,29 +1,26 @@
#pragma once
#include <cstring>
#include <csignal>
#include <cstring>
#include <functional>
#include <mutex>
#include "logging/oxen_logger.h"
#ifdef _WIN32
#include "windows.h"
#endif
namespace tools {
/*! \brief Defines a singleton signal handler for win32 and *nix
*/
class signal_handler
{
class signal_handler {
public:
/*! \brief installs a signal handler */
template <typename T>
static bool install(T t)
{
static bool install(T t) {
#ifdef _WIN32
bool r = TRUE == ::SetConsoleCtrlHandler(&win_handler, TRUE);
if (r)
{
if (r) {
m_handler = t;
}
return r;
@ -44,30 +41,28 @@ namespace tools {
private:
#if defined(WIN32)
/*! \brief Handler for win */
static BOOL WINAPI win_handler(DWORD type)
{
if (CTRL_C_EVENT == type || CTRL_BREAK_EVENT == type)
{
static BOOL WINAPI win_handler(DWORD type) {
if (CTRL_C_EVENT == type || CTRL_BREAK_EVENT == type) {
handle_signal(type);
}
else
{
log::info(globallogcat, fg(fmt::terminal_color::red), "Got control signal {}. Exiting without saving...", type);
} else {
log::info(
globallogcat,
fg(fmt::terminal_color::red),
"Got control signal {}. Exiting without saving...",
type);
return FALSE;
}
return TRUE;
}
#else
/*! \brief handler for NIX */
static void posix_handler(int type)
{
static void posix_handler(int type) {
handle_signal(type);
}
#endif
/*! \brief calles m_handler */
static void handle_signal(int type)
{
static void handle_signal(int type) {
static std::mutex m_mutex;
std::unique_lock lock{m_mutex};
m_handler(type);
@ -77,4 +72,4 @@ namespace tools {
static inline std::function<void(int)> m_handler;
};
}
} // namespace tools

72
src/common/spawn.cpp
View file

@ -27,30 +27,28 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <errno.h>
#include <unistd.h>
#include <sys/types.h>
#include <unistd.h>
#ifdef _WIN32
#include <windows.h>
#else
#include <sys/wait.h>
#include <signal.h>
#include <sys/wait.h>
#endif
#include "epee/misc_log_ex.h"
#include "logging/oxen_logger.h"
#include "util.h"
#include "spawn.h"
#include "oxen.h"
#include "spawn.h"
#include "string_util.h"
#include "util.h"
namespace tools
{
namespace tools {
static auto logcat = log::Cat("spawn");
#ifndef _WIN32
static void closefrom(int fd)
{
static void closefrom(int fd) {
#if defined __FreeBSD__ || defined __OpenBSD__ || defined __NetBSD__ || defined __DragonFly__
::closefrom(fd);
#else
@ -60,8 +58,7 @@ static void closefrom(int fd)
#else
const int MAX_FDS = 65536;
#endif
while (fd < MAX_FDS)
{
while (fd < MAX_FDS) {
close(fd);
++fd;
}
@ -69,9 +66,7 @@ static void closefrom(int fd)
}
#endif
int spawn(const fs::path& filename, const std::vector<std::string>& args, bool wait)
{
int spawn(const fs::path& filename, const std::vector<std::string>& args, bool wait) {
#ifdef _WIN32
std::string joined = tools::join(" ", args);
char* commandLine = !joined.empty() ? &joined[0] : nullptr;
@ -81,8 +76,17 @@ int spawn(const fs::path& filename, const std::vector<std::string>& args, bool w
// This .string() is wrong for non-ascii paths, but if we switch to CreateProcessW and use
// .c_str() directly our commandLine argument will not be accepted (because it then has to be a
// wchar_t* but out input is utf-8). Shame on you for this garbage API, Windows.
if (!CreateProcessA(filename.string().c_str(), commandLine, nullptr, nullptr, false, 0, nullptr, nullptr, &si, &pi))
{
if (!CreateProcessA(
filename.string().c_str(),
commandLine,
nullptr,
nullptr,
false,
0,
nullptr,
nullptr,
&si,
&pi)) {
log::error(logcat, "CreateProcess failed. Error code {}", GetLastError());
return -1;
}
@ -92,21 +96,22 @@ int spawn(const fs::path& filename, const std::vector<std::string>& args, bool w
CloseHandle(pi.hProcess);
};
if (!wait)
{
if (!wait) {
return 0;
}
DWORD result = WaitForSingleObject(pi.hProcess, INFINITE);
if (result != WAIT_OBJECT_0)
{
log::error(logcat, "WaitForSingleObject failed. Result {}, error code {}", result, GetLastError());
if (result != WAIT_OBJECT_0) {
log::error(
logcat,
"WaitForSingleObject failed. Result {}, error code {}",
result,
GetLastError());
return -1;
}
DWORD exitCode;
if (!GetExitCodeProcess(pi.hProcess, &exitCode))
{
if (!GetExitCodeProcess(pi.hProcess, &exitCode)) {
log::error(logcat, "GetExitCodeProcess failed. Error code {}", GetLastError());
return -1;
}
@ -120,15 +125,13 @@ int spawn(const fs::path& filename, const std::vector<std::string>& args, bool w
argv[args.size()] = NULL;
pid_t pid = fork();
if (pid < 0)
{
if (pid < 0) {
log::error(logcat, "Error forking: {}", strerror(errno));
return -1;
}
// child
if (pid == 0)
{
if (pid == 0) {
tools::closefrom(3);
close(0);
char* envp[] = {NULL};
@ -138,29 +141,24 @@ int spawn(const fs::path& filename, const std::vector<std::string>& args, bool w
}
// parent
if (pid > 0)
{
if (!wait)
{
if (pid > 0) {
if (!wait) {
signal(SIGCHLD, SIG_IGN);
return 0;
}
while (1)
{
while (1) {
int wstatus = 0;
pid_t w = waitpid(pid, &wstatus, WUNTRACED | WCONTINUED);
if (w < 0) {
log::error(logcat, "Error waiting for child: {}", strerror(errno));
return -1;
}
if (WIFEXITED(wstatus))
{
if (WIFEXITED(wstatus)) {
log::info(logcat, "Child exited with {}", WEXITSTATUS(wstatus));
return WEXITSTATUS(wstatus);
}
if (WIFSIGNALED(wstatus))
{
if (WIFSIGNALED(wstatus)) {
log::info(logcat, "Child killed by {}", WEXITSTATUS(wstatus));
return WEXITSTATUS(wstatus);
}
@ -171,4 +169,4 @@ int spawn(const fs::path& filename, const std::vector<std::string>& args, bool w
#endif
}
}
} // namespace tools

6
src/common/spawn.h
View file

@ -27,12 +27,12 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <vector>
#include <string>
#include <vector>
#include "common/fs.h"
namespace tools
{
namespace tools {
int spawn(const fs::path& filename, const std::vector<std::string>& args, bool wait);

27
src/common/string_util.cpp
View file

@ -1,25 +1,24 @@
#include "string_util.h"
#include <fmt/core.h>
#include <cassert>
namespace tools {
using namespace std::literals;
std::vector<std::string_view> split(std::string_view str, const std::string_view delim, bool trim)
{
std::vector<std::string_view> split(std::string_view str, const std::string_view delim, bool trim) {
std::vector<std::string_view> results;
// Special case for empty delimiter: splits on each character boundary:
if (delim.empty())
{
if (delim.empty()) {
results.reserve(str.size());
for (size_t i = 0; i < str.size(); i++)
results.emplace_back(str.data() + i, 1);
return results;
}
for (size_t pos = str.find(delim); pos != std::string_view::npos; pos = str.find(delim))
{
for (size_t pos = str.find(delim); pos != std::string_view::npos; pos = str.find(delim)) {
if (!trim || !results.empty() || pos > 0)
results.push_back(str.substr(0, pos));
str.remove_prefix(pos + delim.size());
@ -32,13 +31,13 @@ std::vector<std::string_view> split(std::string_view str, const std::string_view
return results;
}
std::vector<std::string_view> split_any(std::string_view str, const std::string_view delims, bool trim)
{
std::vector<std::string_view> split_any(
std::string_view str, const std::string_view delims, bool trim) {
if (delims.empty())
return split(str, delims, trim);
std::vector<std::string_view> results;
for (size_t pos = str.find_first_of(delims); pos != std::string_view::npos; pos = str.find_first_of(delims))
{
for (size_t pos = str.find_first_of(delims); pos != std::string_view::npos;
pos = str.find_first_of(delims)) {
if (!trim || !results.empty() || pos > 0)
results.push_back(str.substr(0, pos));
size_t until = str.find_first_not_of(delims, pos + 1);
@ -55,8 +54,7 @@ std::vector<std::string_view> split_any(std::string_view str, const std::string_
return results;
}
void trim(std::string_view& s)
{
void trim(std::string_view& s) {
constexpr auto simple_whitespace = " \t\r\n"sv;
auto pos = s.find_first_not_of(simple_whitespace);
if (pos == std::string_view::npos) { // whole string is whitespace
@ -69,8 +67,7 @@ void trim(std::string_view& s)
s.remove_suffix(s.size() - (pos + 1));
}
std::string lowercase_ascii_string(std::string_view src)
{
std::string lowercase_ascii_string(std::string_view src) {
std::string result;
result.reserve(src.size());
for (char ch : src)
@ -114,4 +111,4 @@ std::string friendly_duration(std::chrono::nanoseconds dur) {
return friendly;
}
}
} // namespace tools

49
src/common/string_util.h
View file

@ -1,12 +1,14 @@
#pragma once
#include <string_view>
#include <vector>
#include <cstring>
#include <iterator>
#include <fmt/format.h>
#include <cassert>
#include <charconv>
#include <chrono>
#include <cassert>
#include <fmt/format.h>
#include <cstring>
#include <iterator>
#include <string_view>
#include <vector>
#include "epee/span.h" // epee
namespace tools {
@ -16,7 +18,9 @@ using namespace std::literals;
/// Returns true if the first string is equal to the second string, compared case-insensitively.
inline bool string_iequal(std::string_view s1, std::string_view s2) {
return std::equal(s1.begin(), s1.end(), s2.begin(), s2.end(), [](char a, char b) {
return std::tolower(static_cast<unsigned char>(a)) == std::tolower(static_cast<unsigned char>(b)); });
return std::tolower(static_cast<unsigned char>(a)) ==
std::tolower(static_cast<unsigned char>(b));
});
}
/// Returns true if the first string matches any of the given strings case-insensitively. Arguments
@ -49,7 +53,8 @@ inline bool ends_with(std::string_view str, std::string_view suffix) {
/// auto v = split("-a--b--", "-"); // v is {"", "a", "", "b", "", ""}
/// auto v = split("-a--b--", "-", true); // v is {"a", "", "b"}
///
std::vector<std::string_view> split(std::string_view str, std::string_view delim, bool trim = false);
std::vector<std::string_view> split(
std::string_view str, std::string_view delim, bool trim = false);
/// Splits a string on any 1 or more of the given delimiter characters and returns a vector of
/// string_view's pointing into the pieces of the original string. If delims is empty this works
@ -57,7 +62,8 @@ std::vector<std::string_view> split(std::string_view str, std::string_view delim
/// pieces).
///
/// auto v = split_any("abcdedf", "dcx"); // v is {"ab", "e", "f"}
std::vector<std::string_view> split_any(std::string_view str, std::string_view delims, bool trim = false);
std::vector<std::string_view> split_any(
std::string_view str, std::string_view delims, bool trim = false);
/// Joins [begin, end) with a delimiter and returns the resulting string. Elements can be anything
/// that can be formatted. Semi-deprecated: this just uses fmt to join.
@ -86,7 +92,8 @@ std::string join_transform(std::string_view delimiter, It begin, It end, UnaryOp
/// Wrapper around the above that takes a container and passes c.begin(), c.end().
template <typename Container, typename UnaryOperation>
std::string join_transform(std::string_view delimiter, const Container& c, UnaryOperation&& transform) {
std::string join_transform(
std::string_view delimiter, const Container& c, UnaryOperation&& transform) {
return join_transform(delimiter, c.begin(), c.end(), std::forward<UnaryOperation>(transform));
}
@ -133,8 +140,9 @@ bool parse_int(const std::string_view str, T& value, int base = 10) {
/// that aren't C++-trivial but are still designed to be accessed this way.
template <typename T>
std::string_view view_guts(const T& val) {
static_assert((std::is_standard_layout_v<T> && std::has_unique_object_representations_v<T>)
|| epee::is_byte_spannable<T>,
static_assert(
(std::is_standard_layout_v<T> && std::has_unique_object_representations_v<T>) ||
epee::is_byte_spannable<T>,
"cannot safely access non-trivial class as string_view");
return {reinterpret_cast<const char*>(&val), sizeof(val)};
}
@ -148,8 +156,9 @@ std::string copy_guts(const T& val) {
/// Function to reverse the above view_guts
template <typename T>
T make_from_guts(std::string_view s) {
static_assert((std::is_standard_layout_v<T> && std::has_unique_object_representations_v<T>)
|| epee::is_byte_spannable<T>,
static_assert(
(std::is_standard_layout_v<T> && std::has_unique_object_representations_v<T>) ||
epee::is_byte_spannable<T>,
"cannot safely reconstitute a non-trivial class from data");
if (s.size() != sizeof(T))
throw std::runtime_error("Cannot reconstitute type: wrong type content size");
@ -166,13 +175,13 @@ std::string friendly_duration(std::chrono::nanoseconds dur);
/// Converts a duration into a shorter, single-unit fractional display such as `42.3min`
std::string short_duration(std::chrono::duration<double> dur);
/// Given an array of string arguments, look for strings of the format <prefix><value> and return <value>
/// Returns empty string view if not found.
/// Given an array of string arguments, look for strings of the format <prefix><value> and return
/// <value> Returns empty string view if not found.
template <typename It>
std::string_view find_prefixed_value(It begin, It end, std::string_view prefix)
{
std::string_view find_prefixed_value(It begin, It end, std::string_view prefix) {
auto it = std::find_if(begin, end, [&](const auto& s) { return starts_with(s, prefix); });
if (it == end) return {};
if (it == end)
return {};
return std::string_view{*it}.substr(prefix.size());
}
}
} // namespace tools

48
src/common/threadpool.cpp
View file

@ -25,18 +25,17 @@
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "epee/misc_log_ex.h"
#include "logging/oxen_logger.h"
#include "common/threadpool.h"
#include "cryptonote_config.h"
#include "common/util.h"
#include "cryptonote_config.h"
#include "epee/misc_log_ex.h"
#include "logging/oxen_logger.h"
static thread_local int depth = 0;
static thread_local bool is_leaf = false;
namespace tools
{
namespace tools {
threadpool::threadpool(unsigned int max_threads) : running(true), active(0) {
create(max_threads);
@ -47,22 +46,21 @@ threadpool::~threadpool() {
}
void threadpool::destroy() {
try
{
try {
const std::unique_lock lock{mutex};
running = false;
has_work.notify_all();
}
catch (...)
{
} catch (...) {
// if the lock throws, we're just do it without a lock and hope,
// since the alternative is terminate
running = false;
has_work.notify_all();
}
for (size_t i = 0; i < threads.size(); i++) {
try { threads[i].join(); }
catch (...) { /* ignore */ }
try {
threads[i].join();
} catch (...) { /* ignore */
}
}
threads.clear();
}
@ -108,21 +106,17 @@ unsigned int threadpool::get_max_concurrency() const {
return max;
}
threadpool::waiter::~waiter()
{
try
{
threadpool::waiter::~waiter() {
try {
std::unique_lock lock{mt};
if (num)
log::error(globallogcat, "wait should have been called before waiter dtor - waiting now");
log::error(
globallogcat, "wait should have been called before waiter dtor - waiting now");
} catch (...) { /* ignore */
}
catch (...) { /* ignore */ }
try
{
try {
wait(NULL);
}
catch (const std::exception &e)
{
} catch (const std::exception& e) {
/* ignored */
}
}
@ -151,13 +145,13 @@ void threadpool::run(bool flush) {
std::unique_lock lock{mutex};
while (running) {
entry e;
while(queue.empty() && running)
{
while (queue.empty() && running) {
if (flush)
return;
has_work.wait(lock);
}
if (!running) break;
if (!running)
break;
active++;
e = std::move(queue.front());
@ -175,4 +169,4 @@ void threadpool::run(bool flush) {
active--;
}
}
}
} // namespace tools

19
src/common/threadpool.h
View file

@ -27,21 +27,19 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <condition_variable>
#include <cstddef>
#include <functional>
#include <utility>
#include <vector>
#include <deque>
#include <functional>
#include <mutex>
#include <stdexcept>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <utility>
#include <vector>
namespace tools
{
namespace tools {
//! A global thread pool
class threadpool
{
class threadpool {
public:
static threadpool& getInstance() {
static threadpool instance;
@ -57,6 +55,7 @@ public:
std::mutex mt;
std::condition_variable cv;
int num;
public:
void inc();
void dec();
@ -98,4 +97,4 @@ public:
void run(bool flush = false);
};
}
} // namespace tools

79
src/common/unordered_containers_boost_serialization.h
View file

@ -34,30 +34,29 @@
#include <unordered_map>
#include <unordered_set>
namespace boost
{
namespace serialization
{
namespace boost { namespace serialization {
template <class Archive, class h_key, class hval>
inline void save(Archive &a, const std::unordered_map<h_key, hval> &x, const boost::serialization::version_type ver)
{
inline void save(
Archive& a,
const std::unordered_map<h_key, hval>& x,
const boost::serialization::version_type ver) {
size_t s = x.size();
a << s;
for(auto& v: x)
{
for (auto& v : x) {
a << v.first;
a << v.second;
}
}
template <class Archive, class h_key, class hval>
inline void load(Archive &a, std::unordered_map<h_key, hval> &x, const boost::serialization::version_type ver)
{
inline void load(
Archive& a,
std::unordered_map<h_key, hval>& x,
const boost::serialization::version_type ver) {
x.clear();
size_t s = 0;
a >> s;
for(size_t i = 0; i != s; i++)
{
for (size_t i = 0; i != s; i++) {
h_key k;
hval v;
a >> k;
@ -66,27 +65,28 @@ namespace boost
}
}
template <class Archive, class h_key, class hval>
inline void save(Archive &a, const std::unordered_multimap<h_key, hval> &x, const boost::serialization::version_type ver)
{
inline void save(
Archive& a,
const std::unordered_multimap<h_key, hval>& x,
const boost::serialization::version_type ver) {
size_t s = x.size();
a << s;
for(auto& v: x)
{
for (auto& v : x) {
a << v.first;
a << v.second;
}
}
template <class Archive, class h_key, class hval>
inline void load(Archive &a, std::unordered_multimap<h_key, hval> &x, const boost::serialization::version_type ver)
{
inline void load(
Archive& a,
std::unordered_multimap<h_key, hval>& x,
const boost::serialization::version_type ver) {
x.clear();
size_t s = 0;
a >> s;
for(size_t i = 0; i != s; i++)
{
for (size_t i = 0; i != s; i++) {
h_key k;
hval v;
a >> k;
@ -95,49 +95,50 @@ namespace boost
}
}
template <class Archive, class hval>
inline void save(Archive &a, const std::unordered_set<hval> &x, const boost::serialization::version_type ver)
{
inline void save(
Archive& a,
const std::unordered_set<hval>& x,
const boost::serialization::version_type ver) {
size_t s = x.size();
a << s;
for(auto& v: x)
{
for (auto& v : x) {
a << v;
}
}
template <class Archive, class hval>
inline void load(Archive &a, std::unordered_set<hval> &x, const boost::serialization::version_type ver)
{
inline void load(
Archive& a, std::unordered_set<hval>& x, const boost::serialization::version_type ver) {
x.clear();
size_t s = 0;
a >> s;
for(size_t i = 0; i != s; i++)
{
for (size_t i = 0; i != s; i++) {
hval v;
a >> v;
x.insert(v);
}
}
template <class Archive, class h_key, class hval>
inline void serialize(Archive &a, std::unordered_map<h_key, hval> &x, const boost::serialization::version_type ver)
{
inline void serialize(
Archive& a,
std::unordered_map<h_key, hval>& x,
const boost::serialization::version_type ver) {
split_free(a, x, ver);
}
template <class Archive, class h_key, class hval>
inline void serialize(Archive &a, std::unordered_multimap<h_key, hval> &x, const boost::serialization::version_type ver)
{
inline void serialize(
Archive& a,
std::unordered_multimap<h_key, hval>& x,
const boost::serialization::version_type ver) {
split_free(a, x, ver);
}
template <class Archive, class hval>
inline void serialize(Archive &a, std::unordered_set<hval> &x, const boost::serialization::version_type ver)
{
inline void serialize(
Archive& a, std::unordered_set<hval>& x, const boost::serialization::version_type ver) {
split_free(a, x, ver);
}
}
}
}} // namespace boost::serialization

114
src/common/util.cpp
View file

@ -29,41 +29,39 @@
//
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
#include <chrono>
#include <string>
#include <iomanip>
#include <thread>
#include "util.h"
#include <fmt/chrono.h>
#include <fmt/color.h>
#include <chrono>
#include <iomanip>
#include <string>
#include <thread>
#include "crypto/crypto.h"
#include "epee/misc_log_ex.h"
#include "epee/readline_buffer.h"
#include "epee/string_tools.h"
#include "epee/wipeable_string.h"
#include "crypto/crypto.h"
#include "util.h"
#include "epee/readline_buffer.h"
#include "epee/misc_log_ex.h"
#include "i18n.h"
#include "logging/oxen_logger.h"
#include "string_util.h"
#include "i18n.h"
#ifdef __GLIBC__
#include <sys/resource.h>
#include <gnu/libc-version.h>
#include <sys/resource.h>
#endif
namespace tools
{
namespace tools {
static auto logcat = log::Cat("util");
bool disable_core_dumps()
{
bool disable_core_dumps() {
#ifdef __GLIBC__
// disable core dumps in release mode
struct rlimit rlimit;
rlimit.rlim_cur = rlimit.rlim_max = 0;
if (setrlimit(RLIMIT_CORE, &rlimit))
{
if (setrlimit(RLIMIT_CORE, &rlimit)) {
log::warning(logcat, "Failed to disable core dumps");
return false;
}
@ -71,12 +69,10 @@ namespace tools
return true;
}
ssize_t get_lockable_memory()
{
ssize_t get_lockable_memory() {
#ifdef __GLIBC__
struct rlimit rlim;
if (getrlimit(RLIMIT_MEMLOCK, &rlim) < 0)
{
if (getrlimit(RLIMIT_MEMLOCK, &rlim) < 0) {
log::error(logcat, "Failed to determine the lockable memory limit");
return -1;
}
@ -86,24 +82,25 @@ namespace tools
#endif
}
bool on_startup()
{
bool on_startup() {
#ifdef __GLIBC__
const char* ver = ::gnu_get_libc_version();
if (!strcmp(ver, "2.25"))
log::warning(logcat, fg(fmt::terminal_color::red), "Running with glibc {}, hangs may occur - change glibc version if possible", ver);
log::warning(
logcat,
fg(fmt::terminal_color::red),
"Running with glibc {}, hangs may occur - change glibc version if possible",
ver);
#endif
return true;
}
namespace
{
namespace {
std::mutex max_concurrency_lock;
unsigned max_concurrency = std::thread::hardware_concurrency();
}
} // namespace
void set_max_concurrency(unsigned n)
{
void set_max_concurrency(unsigned n) {
if (n < 1)
n = std::thread::hardware_concurrency();
unsigned hwc = std::thread::hardware_concurrency();
@ -113,22 +110,21 @@ namespace tools
max_concurrency = n;
}
unsigned get_max_concurrency()
{
unsigned get_max_concurrency() {
std::lock_guard lock{max_concurrency_lock};
return max_concurrency;
}
bool is_local_address(const std::string &address)
{
return address == "localhost"sv
|| (tools::starts_with(address, "127."sv) && address.find_first_not_of("0123456789."sv) == std::string::npos)
|| address == "::1"sv
|| address == "[::1]"sv; // There are other uncommon ways to specify localhost (e.g. 0::1, ::0001) but don't worry about them.
bool is_local_address(const std::string& address) {
return address == "localhost"sv ||
(tools::starts_with(address, "127."sv) &&
address.find_first_not_of("0123456789."sv) == std::string::npos) ||
address == "::1"sv ||
address == "[::1]"sv; // There are other uncommon ways to specify localhost (e.g. 0::1,
// ::0001) but don't worry about them.
}
int vercmp(std::string_view v0, std::string_view v1)
{
int vercmp(std::string_view v0, std::string_view v1) {
auto f0 = tools::split_any(v0, ".-");
auto f1 = tools::split_any(v1, ".-");
const auto max = std::max(f0.size(), f1.size());
@ -147,28 +143,30 @@ namespace tools
return 0;
}
std::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str)
{
std::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str) {
auto pos = str.find(":");
bool r = pos != std::string::npos;
uint32_t major;
r = r && epee::string_tools::get_xtype_from_string(major, str.substr(0, pos));
uint32_t minor;
r = r && epee::string_tools::get_xtype_from_string(minor, str.substr(pos + 1));
if (r)
{
if (r) {
return std::make_pair(major, minor);
}
else
{
} else {
return std::nullopt;
}
}
#ifdef _WIN32
std::string input_line_win()
{
HANDLE hConIn = CreateFileW(L"CONIN$", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, 0, nullptr);
std::string input_line_win() {
HANDLE hConIn = CreateFileW(
L"CONIN$",
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE,
nullptr,
OPEN_EXISTING,
0,
nullptr);
DWORD oldMode;
FlushConsoleInputBuffer(hConIn);
@ -192,15 +190,13 @@ namespace tools
}
#endif
std::string get_human_readable_timestamp(std::time_t t)
{
std::string get_human_readable_timestamp(std::time_t t) {
if (t < 1234567890)
return "<unknown>";
return "{:%Y-%m-%d %H:%M:%S} UTC"_format(fmt::gmtime(t));
}
std::string get_human_readable_timespan(std::chrono::seconds seconds)
{
std::string get_human_readable_timespan(std::chrono::seconds seconds) {
uint64_t ts = seconds.count();
if (ts < 60)
return std::to_string(ts) + tr(" seconds");
@ -215,9 +211,9 @@ namespace tools
return tr("a long time");
}
std::string get_human_readable_bytes(uint64_t bytes)
{
if (bytes < 1000) return std::to_string(bytes) + " B";
std::string get_human_readable_bytes(uint64_t bytes) {
if (bytes < 1000)
return std::to_string(bytes) + " B";
constexpr std::array units{" kB", " MB", " GB", " TB"};
double b = bytes;
for (const auto& suffix : units) {
@ -233,8 +229,8 @@ namespace tools
// Calculate a "sync weight" over ranges of blocks in the blockchain, suitable for
// calculating sync time estimates
uint64_t cumulative_block_sync_weight(cryptonote::network_type nettype, uint64_t start_block, uint64_t num_blocks)
{
uint64_t cumulative_block_sync_weight(
cryptonote::network_type nettype, uint64_t start_block, uint64_t num_blocks) {
// No detailed data available except for Mainnet: Give back the number of blocks
// as a very simple and non-varying block sync weight for ranges of Testnet and
// Devnet blocks
@ -310,4 +306,4 @@ namespace tools
return weight;
#endif
}
}
} // namespace tools

47
src/common/util.h
View file

@ -31,10 +31,11 @@
#pragma once
#include <oxenc/endian.h>
#include <optional>
#include <memory>
#include <string>
#include <chrono>
#include <memory>
#include <optional>
#include <string>
#include "crypto/hash.h"
#include "cryptonote_config.h"
@ -45,8 +46,7 @@
*
*
*/
namespace tools
{
namespace tools {
bool disable_core_dumps();
bool on_startup();
@ -57,7 +57,10 @@ namespace tools
unsigned get_max_concurrency();
bool is_local_address(const std::string& address);
int vercmp(std::string_view v0, std::string_view v1); // returns < 0, 0, > 0, similar to strcmp, but more human friendly than lexical - does not attempt to validate
int vercmp(
std::string_view v0,
std::string_view v1); // returns < 0, 0, > 0, similar to strcmp, but more human friendly
// than lexical - does not attempt to validate
std::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str);
@ -69,15 +72,15 @@ namespace tools
std::string get_human_readable_timespan(std::chrono::seconds seconds);
std::string get_human_readable_bytes(uint64_t bytes);
template <typename Duration, std::enable_if_t<!std::is_same_v<Duration, std::chrono::seconds>, int> = 0>
std::string get_human_readable_timespan(Duration d)
{
template <
typename Duration,
std::enable_if_t<!std::is_same_v<Duration, std::chrono::seconds>, int> = 0>
std::string get_human_readable_timespan(Duration d) {
return get_human_readable_timespan(std::chrono::duration_cast<std::chrono::seconds>(d));
}
template <typename Duration>
constexpr uint64_t to_seconds(Duration d)
{
constexpr uint64_t to_seconds(Duration d) {
return std::chrono::duration_cast<std::chrono::seconds>(d).count();
}
@ -96,7 +99,8 @@ namespace tools
static std::false_type check(...);
};
template <typename T>
constexpr bool is_crypto_bytes_derived = decltype(crypto_bytes_base_helper::check((T*) nullptr))::value;
constexpr bool is_crypto_bytes_derived =
decltype(crypto_bytes_base_helper::check((T*)nullptr))::value;
// Copy the data out of a crypto::bytes<N, true>-derived type.
template <typename T, std::enable_if_t<is_crypto_bytes_derived<T>, int> = 0>
@ -106,11 +110,15 @@ namespace tools
}
// Copy a class byte-for-byte (but only if it is standard layout and has byte alignment)
template <typename T, std::enable_if_t<std::is_class<T>::value && !is_crypto_bytes_derived<T>, int> = 0>
template <
typename T,
std::enable_if_t<std::is_class<T>::value && !is_crypto_bytes_derived<T>, int> = 0>
void memcpy_one(char*& dest, const T& t) {
// We don't *actually* require byte alignment here but it's quite possibly an error (i.e.
// passing in a type containing integer members) so disallow it.
static_assert(std::is_trivially_copyable<T>::value && alignof(T) == 1, "memcpy_le() may only be used on simple (1-byte alignment) struct types");
static_assert(
std::is_trivially_copyable<T>::value && alignof(T) == 1,
"memcpy_le() may only be used on simple (1-byte alignment) struct types");
std::memcpy(dest, &t, sizeof(T));
dest += sizeof(T);
}
@ -126,9 +134,9 @@ namespace tools
constexpr size_t memcpy_size = sizeof(T);
template <typename T>
inline constexpr size_t memcpy_size<T, std::enable_if_t<is_crypto_bytes_derived<T>>>
= T::size();
}
inline constexpr size_t memcpy_size<T, std::enable_if_t<is_crypto_bytes_derived<T>>> =
T::size();
} // namespace detail
// Does a memcpy of one or more values into a char array; for any given values that are basic
// integer types the value is first converted from native to little-endian representation (if
@ -153,10 +161,11 @@ namespace tools
template <typename Enum>
constexpr Enum enum_top = static_cast<Enum>(enum_count<Enum> - 1);
uint64_t cumulative_block_sync_weight(cryptonote::network_type nettype, uint64_t start_block, uint64_t num_blocks);
uint64_t cumulative_block_sync_weight(
cryptonote::network_type nettype, uint64_t start_block, uint64_t num_blocks);
template <typename T, typename... Any>
constexpr bool equals_any(const T& v, const Any&... any) {
return (... || (v == any));
}
}
} // namespace tools

45
src/common/varint.h
View file

@ -31,10 +31,10 @@
#pragma once
#include <type_traits>
#include <cassert>
#include <iterator>
#include <string>
#include <cassert>
#include <type_traits>
/*! \file varint.h
* \brief provides the implementation of varint's
@ -86,7 +86,10 @@ namespace tools {
* could static cast a signed type to an unsigned type, but note that any actual negative values
* (which have a high bit set) will always end up full size since the high bit is set.
*/
template <typename OutputIt, typename T, std::enable_if_t<std::is_integral_v<T> && std::is_unsigned_v<T>, int> = 0>
template <
typename OutputIt,
typename T,
std::enable_if_t<std::is_integral_v<T> && std::is_unsigned_v<T>, int> = 0>
void write_varint(OutputIt&& it, T i) {
#ifndef NDEBUG
size_t count = 0;
@ -105,8 +108,7 @@ namespace tools {
/*! \brief Returns the string that represents the varint
*/
template <typename T>
std::string get_varint_data(const T& v)
{
std::string get_varint_data(const T& v) {
std::string result;
write_varint(std::back_insert_iterator{result}, v);
return result;
@ -116,15 +118,18 @@ namespace tools {
* the position after the varint bytes that we read. Returns the number of bytes read on success,
* or one of the negative EVARIANT_* constants on failure.
*/
template <typename It, typename EndIt, typename T, std::enable_if_t<std::is_integral_v<T> && std::is_unsigned_v<T>, int> = 0>
template <
typename It,
typename EndIt,
typename T,
std::enable_if_t<std::is_integral_v<T> && std::is_unsigned_v<T>, int> = 0>
int read_varint(It&& it, const EndIt& end, T& write) {
constexpr size_t bits = sizeof(T) * 8;
bool more = true;
int read = 0;
write = 0;
for (size_t shift = 0; more && it != end; shift += 7)
{
for (size_t shift = 0; more && it != end; shift += 7) {
auto byte = static_cast<unsigned char>(*it++);
++read;
@ -133,7 +138,8 @@ namespace tools {
if (byte == 0 && shift)
return EVARINT_REPRESENT;
// If we have <= 7 bits of space remaining then the value must fit and must not have a continuation bit
// If we have <= 7 bits of space remaining then the value must fit and must not have a
// continuation bit
if (size_t bits_avail = bits - shift; bits_avail <= 7 && byte >= 1 << bits_avail)
return EVARINT_OVERFLOW;
@ -146,12 +152,21 @@ namespace tools {
}
// Overloads to allow {read,write}_varint to be called with const iterators
template <typename OutputIt, typename T, std::enable_if_t<std::is_integral_v<T> && std::is_unsigned_v<T>, int> = 0>
auto write_varint(const OutputIt& it_, T i) { return write_varint(OutputIt{it_}, i); }
template <typename It, typename T, std::enable_if_t<std::is_integral_v<T> && std::is_unsigned_v<T>, int> = 0>
auto read_varint(const It& it_, const It& end, T& i) { return read_varint(It{it_}, end, i); }
template <
typename OutputIt,
typename T,
std::enable_if_t<std::is_integral_v<T> && std::is_unsigned_v<T>, int> = 0>
auto write_varint(const OutputIt& it_, T i) {
return write_varint(OutputIt{it_}, i);
}
template <
typename It,
typename T,
std::enable_if_t<std::is_integral_v<T> && std::is_unsigned_v<T>, int> = 0>
auto read_varint(const It& it_, const It& end, T& i) {
return read_varint(It{it_}, end, i);
}
/*! \brief reads the varint from an encoded string into `write`. Returns the number of bytes
* consumed, or an error value (as above).
@ -160,4 +175,4 @@ namespace tools {
int read_varint(std::string_view s, T& write) {
return read_varint(s.begin(), s.end(), write);
}
}
} // namespace tools

135
src/crypto/aesb.c
View file

@ -19,11 +19,11 @@ Issue Date: 20/12/2007
*/
#include <stdint.h>
#include "epee/int-util.h"
#if defined(__cplusplus)
extern "C"
{
extern "C" {
#endif
#define TABLE_ALIGN 32
@ -47,60 +47,83 @@ extern "C"
#define s(x, c) x[c]
#define si(y, x, c) (s(y, c) = word_in(x, c))
#define so(y, x, c) word_out(y, c, s(x, c))
#define state_in(y,x) si(y,x,0); si(y,x,1); si(y,x,2); si(y,x,3)
#define state_out(y,x) so(y,x,0); so(y,x,1); so(y,x,2); so(y,x,3)
#define round(rm,y,x,k) rm(y,x,k,0); rm(y,x,k,1); rm(y,x,k,2); rm(y,x,k,3)
#define state_in(y, x) \
si(y, x, 0); \
si(y, x, 1); \
si(y, x, 2); \
si(y, x, 3)
#define state_out(y, x) \
so(y, x, 0); \
so(y, x, 1); \
so(y, x, 2); \
so(y, x, 3)
#define round(rm, y, x, k) \
rm(y, x, k, 0); \
rm(y, x, k, 1); \
rm(y, x, k, 2); \
rm(y, x, k, 3)
#define to_byte(x) ((x)&0xff)
#define bval(x, n) to_byte(SWAP32LE(x) >> (8 * (n)))
#define fwd_var(x, r, c) \
( r == 0 ? ( c == 0 ? s(x,0) : c == 1 ? s(x,1) : c == 2 ? s(x,2) : s(x,3))\
: r == 1 ? ( c == 0 ? s(x,1) : c == 1 ? s(x,2) : c == 2 ? s(x,3) : s(x,0))\
: r == 2 ? ( c == 0 ? s(x,2) : c == 1 ? s(x,3) : c == 2 ? s(x,0) : s(x,1))\
: ( c == 0 ? s(x,3) : c == 1 ? s(x,0) : c == 2 ? s(x,1) : s(x,2)))
(r == 0 ? (c == 0 ? s(x, 0) \
: c == 1 ? s(x, 1) \
: c == 2 ? s(x, 2) \
: s(x, 3)) \
: r == 1 ? (c == 0 ? s(x, 1) \
: c == 1 ? s(x, 2) \
: c == 2 ? s(x, 3) \
: s(x, 0)) \
: r == 2 ? (c == 0 ? s(x, 2) \
: c == 1 ? s(x, 3) \
: c == 2 ? s(x, 0) \
: s(x, 1)) \
: (c == 0 ? s(x, 3) \
: c == 1 ? s(x, 0) \
: c == 2 ? s(x, 1) \
: s(x, 2)))
#define fwd_rnd(y,x,k,c) (s(y,c) = (k)[c] ^ SWAP32LE(four_tables(x,t_use(f,n),fwd_var,rf1,c)))
#define fwd_rnd(y, x, k, c) \
(s(y, c) = (k)[c] ^ SWAP32LE(four_tables(x, t_use(f, n), fwd_var, rf1, c)))
#define sb_data(w) {\
w(0x63), w(0x7c), w(0x77), w(0x7b), w(0xf2), w(0x6b), w(0x6f), w(0xc5),\
w(0x30), w(0x01), w(0x67), w(0x2b), w(0xfe), w(0xd7), w(0xab), w(0x76),\
w(0xca), w(0x82), w(0xc9), w(0x7d), w(0xfa), w(0x59), w(0x47), w(0xf0),\
w(0xad), w(0xd4), w(0xa2), w(0xaf), w(0x9c), w(0xa4), w(0x72), w(0xc0),\
w(0xb7), w(0xfd), w(0x93), w(0x26), w(0x36), w(0x3f), w(0xf7), w(0xcc),\
w(0x34), w(0xa5), w(0xe5), w(0xf1), w(0x71), w(0xd8), w(0x31), w(0x15),\
w(0x04), w(0xc7), w(0x23), w(0xc3), w(0x18), w(0x96), w(0x05), w(0x9a),\
w(0x07), w(0x12), w(0x80), w(0xe2), w(0xeb), w(0x27), w(0xb2), w(0x75),\
w(0x09), w(0x83), w(0x2c), w(0x1a), w(0x1b), w(0x6e), w(0x5a), w(0xa0),\
w(0x52), w(0x3b), w(0xd6), w(0xb3), w(0x29), w(0xe3), w(0x2f), w(0x84),\
w(0x53), w(0xd1), w(0x00), w(0xed), w(0x20), w(0xfc), w(0xb1), w(0x5b),\
w(0x6a), w(0xcb), w(0xbe), w(0x39), w(0x4a), w(0x4c), w(0x58), w(0xcf),\
w(0xd0), w(0xef), w(0xaa), w(0xfb), w(0x43), w(0x4d), w(0x33), w(0x85),\
w(0x45), w(0xf9), w(0x02), w(0x7f), w(0x50), w(0x3c), w(0x9f), w(0xa8),\
w(0x51), w(0xa3), w(0x40), w(0x8f), w(0x92), w(0x9d), w(0x38), w(0xf5),\
w(0xbc), w(0xb6), w(0xda), w(0x21), w(0x10), w(0xff), w(0xf3), w(0xd2),\
w(0xcd), w(0x0c), w(0x13), w(0xec), w(0x5f), w(0x97), w(0x44), w(0x17),\
w(0xc4), w(0xa7), w(0x7e), w(0x3d), w(0x64), w(0x5d), w(0x19), w(0x73),\
w(0x60), w(0x81), w(0x4f), w(0xdc), w(0x22), w(0x2a), w(0x90), w(0x88),\
w(0x46), w(0xee), w(0xb8), w(0x14), w(0xde), w(0x5e), w(0x0b), w(0xdb),\
w(0xe0), w(0x32), w(0x3a), w(0x0a), w(0x49), w(0x06), w(0x24), w(0x5c),\
w(0xc2), w(0xd3), w(0xac), w(0x62), w(0x91), w(0x95), w(0xe4), w(0x79),\
w(0xe7), w(0xc8), w(0x37), w(0x6d), w(0x8d), w(0xd5), w(0x4e), w(0xa9),\
w(0x6c), w(0x56), w(0xf4), w(0xea), w(0x65), w(0x7a), w(0xae), w(0x08),\
w(0xba), w(0x78), w(0x25), w(0x2e), w(0x1c), w(0xa6), w(0xb4), w(0xc6),\
w(0xe8), w(0xdd), w(0x74), w(0x1f), w(0x4b), w(0xbd), w(0x8b), w(0x8a),\
w(0x70), w(0x3e), w(0xb5), w(0x66), w(0x48), w(0x03), w(0xf6), w(0x0e),\
w(0x61), w(0x35), w(0x57), w(0xb9), w(0x86), w(0xc1), w(0x1d), w(0x9e),\
w(0xe1), w(0xf8), w(0x98), w(0x11), w(0x69), w(0xd9), w(0x8e), w(0x94),\
w(0x9b), w(0x1e), w(0x87), w(0xe9), w(0xce), w(0x55), w(0x28), w(0xdf),\
w(0x8c), w(0xa1), w(0x89), w(0x0d), w(0xbf), w(0xe6), w(0x42), w(0x68),\
w(0x41), w(0x99), w(0x2d), w(0x0f), w(0xb0), w(0x54), w(0xbb), w(0x16) }
#define sb_data(w) \
{ \
w(0x63), w(0x7c), w(0x77), w(0x7b), w(0xf2), w(0x6b), w(0x6f), w(0xc5), w(0x30), w(0x01), \
w(0x67), w(0x2b), w(0xfe), w(0xd7), w(0xab), w(0x76), w(0xca), w(0x82), w(0xc9), \
w(0x7d), w(0xfa), w(0x59), w(0x47), w(0xf0), w(0xad), w(0xd4), w(0xa2), w(0xaf), \
w(0x9c), w(0xa4), w(0x72), w(0xc0), w(0xb7), w(0xfd), w(0x93), w(0x26), w(0x36), \
w(0x3f), w(0xf7), w(0xcc), w(0x34), w(0xa5), w(0xe5), w(0xf1), w(0x71), w(0xd8), \
w(0x31), w(0x15), w(0x04), w(0xc7), w(0x23), w(0xc3), w(0x18), w(0x96), w(0x05), \
w(0x9a), w(0x07), w(0x12), w(0x80), w(0xe2), w(0xeb), w(0x27), w(0xb2), w(0x75), \
w(0x09), w(0x83), w(0x2c), w(0x1a), w(0x1b), w(0x6e), w(0x5a), w(0xa0), w(0x52), \
w(0x3b), w(0xd6), w(0xb3), w(0x29), w(0xe3), w(0x2f), w(0x84), w(0x53), w(0xd1), \
w(0x00), w(0xed), w(0x20), w(0xfc), w(0xb1), w(0x5b), w(0x6a), w(0xcb), w(0xbe), \
w(0x39), w(0x4a), w(0x4c), w(0x58), w(0xcf), w(0xd0), w(0xef), w(0xaa), w(0xfb), \
w(0x43), w(0x4d), w(0x33), w(0x85), w(0x45), w(0xf9), w(0x02), w(0x7f), w(0x50), \
w(0x3c), w(0x9f), w(0xa8), w(0x51), w(0xa3), w(0x40), w(0x8f), w(0x92), w(0x9d), \
w(0x38), w(0xf5), w(0xbc), w(0xb6), w(0xda), w(0x21), w(0x10), w(0xff), w(0xf3), \
w(0xd2), w(0xcd), w(0x0c), w(0x13), w(0xec), w(0x5f), w(0x97), w(0x44), w(0x17), \
w(0xc4), w(0xa7), w(0x7e), w(0x3d), w(0x64), w(0x5d), w(0x19), w(0x73), w(0x60), \
w(0x81), w(0x4f), w(0xdc), w(0x22), w(0x2a), w(0x90), w(0x88), w(0x46), w(0xee), \
w(0xb8), w(0x14), w(0xde), w(0x5e), w(0x0b), w(0xdb), w(0xe0), w(0x32), w(0x3a), \
w(0x0a), w(0x49), w(0x06), w(0x24), w(0x5c), w(0xc2), w(0xd3), w(0xac), w(0x62), \
w(0x91), w(0x95), w(0xe4), w(0x79), w(0xe7), w(0xc8), w(0x37), w(0x6d), w(0x8d), \
w(0xd5), w(0x4e), w(0xa9), w(0x6c), w(0x56), w(0xf4), w(0xea), w(0x65), w(0x7a), \
w(0xae), w(0x08), w(0xba), w(0x78), w(0x25), w(0x2e), w(0x1c), w(0xa6), w(0xb4), \
w(0xc6), w(0xe8), w(0xdd), w(0x74), w(0x1f), w(0x4b), w(0xbd), w(0x8b), w(0x8a), \
w(0x70), w(0x3e), w(0xb5), w(0x66), w(0x48), w(0x03), w(0xf6), w(0x0e), w(0x61), \
w(0x35), w(0x57), w(0xb9), w(0x86), w(0xc1), w(0x1d), w(0x9e), w(0xe1), w(0xf8), \
w(0x98), w(0x11), w(0x69), w(0xd9), w(0x8e), w(0x94), w(0x9b), w(0x1e), w(0x87), \
w(0xe9), w(0xce), w(0x55), w(0x28), w(0xdf), w(0x8c), w(0xa1), w(0x89), w(0x0d), \
w(0xbf), w(0xe6), w(0x42), w(0x68), w(0x41), w(0x99), w(0x2d), w(0x0f), w(0xb0), \
w(0x54), w(0xbb), w(0x16) \
}
#define rc_data(w) {\
w(0x01), w(0x02), w(0x04), w(0x08), w(0x10),w(0x20), w(0x40), w(0x80),\
w(0x1b), w(0x36) }
#define rc_data(w) \
{ w(0x01), w(0x02), w(0x04), w(0x08), w(0x10), w(0x20), w(0x40), w(0x80), w(0x1b), w(0x36) }
#define bytes2word(b0, b1, b2, b3) (((uint32_t)(b3) << 24) | \
((uint32_t)(b2) << 16) | ((uint32_t)(b1) << 8) | (b0))
#define bytes2word(b0, b1, b2, b3) \
(((uint32_t)(b3) << 24) | ((uint32_t)(b2) << 16) | ((uint32_t)(b1) << 8) | (b0))
#define h0(x) (x)
#define w0(p) bytes2word(p, 0, 0, 0)
@ -120,7 +143,8 @@ extern "C"
#define f2(x) ((x << 1) ^ (((x >> 7) & 1) * WPOLY))
#define f4(x) ((x << 2) ^ (((x >> 6) & 1) * WPOLY) ^ (((x >> 6) & 2) * WPOLY))
#define f8(x) ((x<<3) ^ (((x>>5) & 1) * WPOLY) ^ (((x>>5) & 2) * WPOLY) ^ (((x>>5) & 4) * WPOLY))
#define f8(x) \
((x << 3) ^ (((x >> 5) & 1) * WPOLY) ^ (((x >> 5) & 2) * WPOLY) ^ (((x >> 5) & 4) * WPOLY))
#define f3(x) (f2(x) ^ x)
#define f9(x) (f8(x) ^ x)
#define fb(x) (f8(x) ^ f2(x) ^ x)
@ -134,10 +158,8 @@ extern "C"
#define d_4(t, n, b, e, f, g, h) LOCAL_ALIGN const t n[4][256] = {b(e), b(f), b(g), b(h)}
#define four_tables(x, tab, vf, rf, c) \
(tab[0][bval(vf(x,0,c),rf(0,c))] \
^ tab[1][bval(vf(x,1,c),rf(1,c))] \
^ tab[2][bval(vf(x,2,c),rf(2,c))] \
^ tab[3][bval(vf(x,3,c),rf(3,c))])
(tab[0][bval(vf(x, 0, c), rf(0, c))] ^ tab[1][bval(vf(x, 1, c), rf(1, c))] ^ \
tab[2][bval(vf(x, 2, c), rf(2, c))] ^ tab[3][bval(vf(x, 3, c), rf(3, c))])
d_4(uint32_t, t_dec(f, n), sb_data, u0, u1, u2, u3);
@ -149,8 +171,7 @@ d_4(uint32_t, t_dec(f,n), sb_data, u0, u1, u2, u3);
#define INLINE
#endif
STATIC INLINE void aesb_single_round(const uint8_t *in, uint8_t *out, uint8_t *expandedKey)
{
STATIC INLINE void aesb_single_round(const uint8_t* in, uint8_t* out, uint8_t* expandedKey) {
uint32_t b0[4], b1[4];
const uint32_t* kp = (uint32_t*)expandedKey;
state_in(b0, in);
@ -160,8 +181,7 @@ STATIC INLINE void aesb_single_round(const uint8_t *in, uint8_t *out, uint8_t *e
state_out(out, b1);
}
STATIC INLINE void aesb_pseudo_round(const uint8_t *in, uint8_t *out, uint8_t *expandedKey)
{
STATIC INLINE void aesb_pseudo_round(const uint8_t* in, uint8_t* out, uint8_t* expandedKey) {
uint32_t b0[4], b1[4];
const uint32_t* kp = (uint32_t*)expandedKey;
state_in(b0, in);
@ -180,7 +200,6 @@ STATIC INLINE void aesb_pseudo_round(const uint8_t *in, uint8_t *out, uint8_t *e
state_out(out, b0);
}
#if defined(__cplusplus)
}
#endif

29
src/crypto/base.h
View file

@ -1,16 +1,19 @@
#pragma once
#include <array>
#include <type_traits>
#include "common/format.h"
#include "common/formattable.h"
#include "common/hex.h"
#include <array>
#include <type_traits>
namespace crypto {
/// constexpr null (all-0) value for various crypto types; use as `crypto::null<crypto::whatever>`.
template <typename T, typename = std::enable_if_t<std::is_standard_layout_v<T> && std::is_default_constructible_v<T>>>
template <
typename T,
typename = std::enable_if_t<
std::is_standard_layout_v<T> && std::is_default_constructible_v<T>>>
constexpr T null{};
// Base type for fixed-byte quantities (points, scalars, signatures, hashes). The bool controls
@ -46,7 +49,9 @@ namespace crypto {
template <typename Left, typename Right, typename = void>
constexpr bool are_comparable_v = false;
template <typename L, typename R>
inline constexpr bool are_comparable_v<L, R, std::enable_if_t<std::is_same_v<L, R> && has_compare_hash_hex<L>>> = true;
inline constexpr bool
are_comparable_v<L, R, std::enable_if_t<std::is_same_v<L, R> && has_compare_hash_hex<L>>> =
true;
template <typename L, typename R, std::enable_if_t<are_comparable_v<L, R>, int> = 0>
bool operator==(const L& left, const R& right) {
@ -62,7 +67,9 @@ namespace crypto {
}
template <typename T, typename = std::enable_if_t<has_compare_hash_hex<T>>>
std::string to_hex_string(const T& val) { return "<{}>"_format(tools::type_to_hex(val)); }
std::string to_hex_string(const T& val) {
return "<{}>"_format(tools::type_to_hex(val));
}
template <typename T>
struct raw_hasher {
@ -71,12 +78,10 @@ namespace crypto {
static_assert(sizeof(T) >= sizeof(size_t));
static_assert(alignof(T) >= sizeof(size_t));
size_t operator()(const T& val) const {
return *reinterpret_cast<const size_t*>(val.data());
}
size_t operator()(const T& val) const { return *reinterpret_cast<const size_t*>(val.data()); }
};
}
} // namespace crypto
template <typename T>
inline constexpr bool formattable::via_to_hex_string<T, std::enable_if_t<crypto::has_compare_hash_hex<T>>>
= true;
inline constexpr bool
formattable::via_to_hex_string<T, std::enable_if_t<crypto::has_compare_hash_hex<T>>> = true;

81
src/crypto/blake256.c
View file

@ -37,17 +37,21 @@
* HMAC is specified by RFC 2104.
*/
#include <string.h>
#include <stdint.h>
#include "epee/memwipe.h"
#include "blake256.h"
#include <stdint.h>
#include <string.h>
#include "epee/memwipe.h"
#define U8TO32(p) \
(((uint32_t)((p)[0]) << 24) | ((uint32_t)((p)[1]) << 16) | \
((uint32_t)((p)[2]) << 8) | ((uint32_t)((p)[3]) ))
(((uint32_t)((p)[0]) << 24) | ((uint32_t)((p)[1]) << 16) | ((uint32_t)((p)[2]) << 8) | \
((uint32_t)((p)[3])))
#define U32TO8(p, v) \
(p)[0] = (uint8_t)((v) >> 24); (p)[1] = (uint8_t)((v) >> 16); \
(p)[2] = (uint8_t)((v) >> 8); (p)[3] = (uint8_t)((v) );
(p)[0] = (uint8_t)((v) >> 24); \
(p)[1] = (uint8_t)((v) >> 16); \
(p)[2] = (uint8_t)((v) >> 8); \
(p)[3] = (uint8_t)((v));
const uint8_t sigma[][16] = {
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
@ -63,21 +67,30 @@ const uint8_t sigma[][16] = {
{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
{14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3},
{11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4},
{ 7, 9, 3, 1,13,12,11,14, 2, 6, 5,10, 4, 0,15, 8}
};
{7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8}};
const uint32_t cst[16] = {
0x243F6A88, 0x85A308D3, 0x13198A2E, 0x03707344,
0xA4093822, 0x299F31D0, 0x082EFA98, 0xEC4E6C89,
0x452821E6, 0x38D01377, 0xBE5466CF, 0x34E90C6C,
0xC0AC29B7, 0xC97C50DD, 0x3F84D5B5, 0xB5470917
};
static const uint8_t padding[] = {
0x80,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
};
0x243F6A88,
0x85A308D3,
0x13198A2E,
0x03707344,
0xA4093822,
0x299F31D0,
0x082EFA98,
0xEC4E6C89,
0x452821E6,
0x38D01377,
0xBE5466CF,
0x34E90C6C,
0xC0AC29B7,
0xC97C50DD,
0x3F84D5B5,
0xB5470917};
static const uint8_t padding[] = {0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
void blake256_compress(state* S, const uint8_t* block) {
uint32_t v[16], m[16], i;
@ -93,8 +106,10 @@ void blake256_compress(state *S, const uint8_t *block) {
v[c] += v[d]; \
v[b] = ROT(v[b] ^ v[c], 7);
for (i = 0; i < 16; ++i) m[i] = U8TO32(block + i * 4);
for (i = 0; i < 8; ++i) v[i] = S->h[i];
for (i = 0; i < 16; ++i)
m[i] = U8TO32(block + i * 4);
for (i = 0; i < 8; ++i)
v[i] = S->h[i];
v[8] = S->s[0] ^ 0x243F6A88;
v[9] = S->s[1] ^ 0x85A308D3;
v[10] = S->s[2] ^ 0x13198A2E;
@ -122,8 +137,10 @@ void blake256_compress(state *S, const uint8_t *block) {
G(1, 6, 11, 12, 10);
}
for (i = 0; i < 16; ++i) S->h[i % 8] ^= v[i];
for (i = 0; i < 8; ++i) S->h[i] ^= S->s[i % 4];
for (i = 0; i < 16; ++i)
S->h[i % 8] ^= v[i];
for (i = 0; i < 8; ++i)
S->h[i] ^= S->s[i % 4];
}
void blake256_init(state* S) {
@ -160,7 +177,8 @@ void blake256_update(state *S, const uint8_t *data, uint64_t datalen) {
if (left && (((datalen >> 3)) >= (unsigned)fill)) {
memcpy((void*)(S->buf + left), (void*)data, fill);
S->t[0] += 512;
if (S->t[0] == 0) S->t[1]++;
if (S->t[0] == 0)
S->t[1]++;
blake256_compress(S, S->buf);
data += fill;
datalen -= (fill << 3);
@ -169,7 +187,8 @@ void blake256_update(state *S, const uint8_t *data, uint64_t datalen) {
while (datalen >= 512) {
S->t[0] += 512;
if (S->t[0] == 0) S->t[1]++;
if (S->t[0] == 0)
S->t[1]++;
blake256_compress(S, data);
data += 64;
datalen -= 512;
@ -191,7 +210,8 @@ void blake224_update(state *S, const uint8_t *data, uint64_t datalen) {
void blake256_final_h(state* S, uint8_t* digest, uint8_t pa, uint8_t pb) {
uint8_t msglen[8];
uint32_t lo = S->t[0] + S->buflen, hi = S->t[1];
if (lo < (unsigned) S->buflen) hi++;
if (lo < (unsigned)S->buflen)
hi++;
U32TO8(msglen + 0, hi);
U32TO8(msglen + 4, lo);
@ -200,7 +220,8 @@ void blake256_final_h(state *S, uint8_t *digest, uint8_t pa, uint8_t pb) {
blake256_update(S, &pa, 8);
} else {
if (S->buflen < 440) { /* enough space to fill the block */
if (S->buflen == 0) S->nullt = 1;
if (S->buflen == 0)
S->nullt = 1;
S->t[0] -= 440 - S->buflen;
blake256_update(S, padding, 440 - S->buflen);
} else { /* need 2 compressions */
@ -339,7 +360,8 @@ void hmac_blake224_final(hmac_state *S, uint8_t *digest) {
}
// keylen = number of bytes; inlen = number of bytes
void hmac_blake256_hash(uint8_t *out, const uint8_t *key, uint64_t keylen, const uint8_t *in, uint64_t inlen) {
void hmac_blake256_hash(
uint8_t* out, const uint8_t* key, uint64_t keylen, const uint8_t* in, uint64_t inlen) {
hmac_state S;
hmac_blake256_init(&S, key, keylen);
hmac_blake256_update(&S, in, inlen * 8);
@ -347,7 +369,8 @@ void hmac_blake256_hash(uint8_t *out, const uint8_t *key, uint64_t keylen, const
}
// keylen = number of bytes; inlen = number of bytes
void hmac_blake224_hash(uint8_t *out, const uint8_t *key, uint64_t keylen, const uint8_t *in, uint64_t inlen) {
void hmac_blake224_hash(
uint8_t* out, const uint8_t* key, uint64_t keylen, const uint8_t* in, uint64_t inlen) {
hmac_state S;
hmac_blake224_init(&S, key, keylen);
hmac_blake224_update(&S, in, inlen * 8);

8
src/crypto/c_threads.h
View file

@ -33,11 +33,15 @@
#include <windows.h>
#define CTHR_MUTEX_TYPE HANDLE
#define CTHR_MUTEX_INIT NULL
#define CTHR_MUTEX_LOCK(x) do { if (x == NULL) { \
#define CTHR_MUTEX_LOCK(x) \
do { \
if (x == NULL) { \
HANDLE p = CreateMutex(NULL, FALSE, NULL); \
if (InterlockedCompareExchangePointer((PVOID*)&x, (PVOID)p, NULL) != NULL) \
CloseHandle(p); \
} WaitForSingleObject(x, INFINITE); } while(0)
} \
WaitForSingleObject(x, INFINITE); \
} while (0)
#define CTHR_MUTEX_UNLOCK(x) ReleaseMutex(x)
#define CTHR_THREAD_TYPE HANDLE
#define CTHR_THREAD_RTYPE void

33
src/crypto/chacha.c
View file

@ -34,23 +34,34 @@ Public domain.
#define PLUSONE(v) (PLUS((v), 1))
#define QUARTERROUND(a, b, c, d) \
a = PLUS(a,b); d = ROTATE(XOR(d,a),16); \
c = PLUS(c,d); b = ROTATE(XOR(b,c),12); \
a = PLUS(a,b); d = ROTATE(XOR(d,a), 8); \
c = PLUS(c,d); b = ROTATE(XOR(b,c), 7);
a = PLUS(a, b); \
d = ROTATE(XOR(d, a), 16); \
c = PLUS(c, d); \
b = ROTATE(XOR(b, c), 12); \
a = PLUS(a, b); \
d = ROTATE(XOR(d, a), 8); \
c = PLUS(c, d); \
b = ROTATE(XOR(b, c), 7);
static const char sigma[] = "expand 32-byte k";
DISABLE_GCC_AND_CLANG_WARNING(strict - aliasing)
static void chacha(unsigned rounds, const void* data, size_t length, const uint8_t* key, const uint8_t* iv, char* cipher) {
static void chacha(
unsigned rounds,
const void* data,
size_t length,
const uint8_t* key,
const uint8_t* iv,
char* cipher) {
uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15;
char* ctarget = 0;
char tmp[64];
int i;
if (!length) return;
if (!length)
return;
j0 = U8TO32_LITTLE(sigma + 0);
j1 = U8TO32_LITTLE(sigma + 4);
@ -137,8 +148,7 @@ static void chacha(unsigned rounds, const void* data, size_t length, const uint8
x15 = XOR(x15, U8TO32_LITTLE((uint8_t*)data + 60));
j12 = PLUSONE(j12);
if (!j12)
{
if (!j12) {
j13 = PLUSONE(j13);
/* stopping at 2^70 bytes per iv is user's responsibility */
}
@ -172,12 +182,11 @@ static void chacha(unsigned rounds, const void* data, size_t length, const uint8
}
}
void chacha8(const void* data, size_t length, const uint8_t* key, const uint8_t* iv, char* cipher)
{
void chacha8(const void* data, size_t length, const uint8_t* key, const uint8_t* iv, char* cipher) {
chacha(8, data, length, key, iv, cipher);
}
void chacha20(const void* data, size_t length, const uint8_t* key, const uint8_t* iv, char* cipher)
{
void chacha20(
const void* data, size_t length, const uint8_t* key, const uint8_t* iv, char* cipher) {
chacha(20, data, length, key, iv, cipher);
}

32
src/crypto/chacha.cpp
View file

@ -1,18 +1,29 @@
#include "chacha.h"
#include "hash.h"
#include "cn_heavy_hash.hpp"
#include "hash.h"
namespace crypto {
static_assert(sizeof(chacha_key) == CHACHA_KEY_SIZE && sizeof(chacha_iv) == CHACHA_IV_SIZE, "Invalid structure size");
static_assert(
sizeof(chacha_key) == CHACHA_KEY_SIZE && sizeof(chacha_iv) == CHACHA_IV_SIZE,
"Invalid structure size");
void chacha8(const void* data, std::size_t length, const chacha_key& key, const chacha_iv& iv, char* cipher) {
void chacha8(
const void* data,
std::size_t length,
const chacha_key& key,
const chacha_iv& iv,
char* cipher) {
chacha8(data, length, key.data(), reinterpret_cast<const uint8_t*>(&iv), cipher);
}
void chacha20(const void* data, std::size_t length, const chacha_key& key, const chacha_iv& iv, char* cipher) {
void chacha20(
const void* data,
std::size_t length,
const chacha_key& key,
const chacha_iv& iv,
char* cipher) {
chacha20(data, length, key.data(), reinterpret_cast<const uint8_t*>(&iv), cipher);
}
@ -21,7 +32,8 @@ namespace crypto {
}
void generate_chacha_key(const void* data, size_t size, chacha_key& key, uint64_t kdf_rounds) {
static_assert(sizeof(chacha_key) <= hash::size(), "Size of hash must be at least that of chacha_key");
static_assert(
sizeof(chacha_key) <= hash::size(), "Size of hash must be at least that of chacha_key");
epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash;
static thread_local cn_heavy_hash_v1 ctx;
ctx.hash(data, size, pwd_hash.data());
@ -30,8 +42,10 @@ namespace crypto {
memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key));
}
void generate_chacha_key_prehashed(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) {
static_assert(sizeof(chacha_key) <= hash::size(), "Size of hash must be at least that of chacha_key");
void generate_chacha_key_prehashed(
const void* data, size_t size, chacha_key& key, uint64_t kdf_rounds) {
static_assert(
sizeof(chacha_key) <= hash::size(), "Size of hash must be at least that of chacha_key");
epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash;
static thread_local cn_heavy_hash_v1 ctx;
ctx.hash(data, size, pwd_hash.data(), true);
@ -40,4 +54,4 @@ namespace crypto {
memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key));
}
}
} // namespace crypto

Some files were not shown because too many files have changed in this diff Show more