oxen-io
/
oxen-core
mirror of https://github.com/oxen-io/oxen-core.git
2
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #1572 from darcys22/logging-refactor 

Logging refactor
This commit is contained in:
Jason Rhinelander 2022-10-18 12:40:04 -03:00 committed by GitHub
parent 0f85299e1f 9792b637fb
commit 099b87d5b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
344 changed files with 27760 additions and 36819 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.drone.jsonnet
View File

@ -144,6 +144,7 @@ local mac_builder(name,
'mkdir build',
'cd build',
'cmake .. -G Ninja -DCMAKE_CXX_FLAGS=-fcolor-diagnostics -DCMAKE_BUILD_TYPE=' + build_type + ' ' +
'-DOXEN_LOGGING_FORCE_SUBMODULES=ON ' +
'-DLOCAL_MIRROR=https://builds.lokinet.dev/deps -DUSE_LTO=' + (if lto then 'ON ' else 'OFF ') +
(if werror then '-DWARNINGS_AS_ERRORS=ON ' else '') +
(if build_tests || run_tests then '-DBUILD_TESTS=ON ' else '') +

12
.gitmodules vendored
View File

@ -7,9 +7,6 @@
[submodule "external/randomx"]
path = external/randomx
url = https://github.com/oxen-io/loki-randomXL
[submodule "external/loki-mq"]
path = external/oxen-mq
url = https://github.com/oxen-io/loki-mq.git
[submodule "external/googletest"]
path = external/googletest
url = https://github.com/google/googletest.git
@ -25,9 +22,6 @@
[submodule "external/ghc-filesystem"]
path = external/ghc-filesystem
url = https://github.com/gulrak/filesystem.git
[submodule "external/fmt"]
path = external/fmt
url = https://github.com/fmtlib/fmt.git
[submodule "external/Catch2"]
path = external/Catch2
url = https://github.com/catchorg/Catch2
@ -43,3 +37,9 @@
[submodule "external/oxen-encoding"]
path = external/oxen-encoding
url = https://github.com/oxen-io/oxen-encoding.git
[submodule "external/oxen-logging"]
path = external/oxen-logging
url = https://github.com/oxen-io/oxen-logging
[submodule "external/oxen-mq"]
path = external/oxen-mq
url = https://github.com/oxen-io/oxen-mq.git

4
CMakeLists.txt
View File

@ -44,7 +44,7 @@ if(CCACHE_PROGRAM)
endforeach()
endif()
cmake_minimum_required(VERSION 3.10)
cmake_minimum_required(VERSION 3.13...3.24)
message(STATUS "CMake version ${CMAKE_VERSION}")
# Has to be set before `project()`, and ignored on non-macos:
@ -299,7 +299,6 @@ if(NOT MANUAL_SUBMODULES)
endif()
check_submodule(external/uWebSockets uSockets)
check_submodule(external/ghc-filesystem)
check_submodule(external/SQLiteCpp)
endif()
endif()
@ -491,7 +490,6 @@ if(NOT BUILD_STATIC_DEPS)
endif()
add_subdirectory(external)
target_compile_definitions(easylogging PRIVATE AUTO_INITIALIZE_EASYLOGGINGPP)
if(USE_DEVICE_TREZOR)
include(CheckTrezor)

4
README.md
View File

@ -1,6 +1,6 @@
# Oxen
Copyright (c) 2018-2021 The Oxen Project.
Copyright (c) 2018-2022 The Oxen Project.
Portions Copyright (c) 2014-2019 The Monero Project.
Portions Copyright (c) 2012-2013 The Cryptonote developers.
@ -40,7 +40,7 @@ library archives (`.a`).
| Dep | Min. version | Vendored | Debian/Ubuntu pkg | Arch pkg | Fedora | Optional | Purpose |
| ------------ | ------------- | -------- | ---------------------- | ------------ | ------------------- | -------- | ---------------- |
| GCC | 8.1.0 | NO | `g++`[1] | `base-devel` | `gcc` | NO | |
| CMake | 3.10 | NO | `cmake` | `cmake` | `cmake` | NO | |
| CMake | 3.13 | NO | `cmake` | `cmake` | `cmake` | NO | |
| pkg-config | any | NO | `pkg-config` | `base-devel` | `pkgconf` | NO | |
| Boost | 1.65 | NO | `libboost-all-dev`[2] | `boost` | `boost-devel` | NO | C++ libraries |
| libzmq | 4.3.0 | YES | `libzmq3-dev` | `zeromq` | `zeromq-devel` | NO | ZeroMQ library |

13
cmake/StaticBuild.cmake
View File

@ -27,11 +27,11 @@ set(READLINE_SOURCE readline-${READLINE_VERSION}.tar.gz)
set(READLINE_HASH SHA512=27790d0461da3093a7fee6e89a51dcab5dc61928ec42e9228ab36493b17220641d5e481ea3d8fee5ee0044c70bf960f55c7d3f1a704cf6b9c42e5c269b797e00
CACHE STRING "readline source hash")
set(SQLITE3_VERSION 3390200 CACHE STRING "sqlite3 version")
set(SQLITE3_VERSION 3390300 CACHE STRING "sqlite3 version")
set(SQLITE3_MIRROR ${LOCAL_MIRROR} https://www.sqlite.org/2022
CACHE STRING "sqlite3 download mirror(s)")
set(SQLITE3_SOURCE sqlite-autoconf-${SQLITE3_VERSION}.tar.gz)
set(SQLITE3_HASH SHA512=c16b50ade3c182d5473014ac0a51e2bb8a5cfc46e532c2bda77ae4d530336e2b57aa4f12dccb6aa2148d60e9289305bf20842ac95dc52f2d31df8eb5f0599de6
set(SQLITE3_HASH SHA512=f5f1f275ca7def6e1971d0152852468b89f2759d6fc2dda42b6e47646d65c7d8c454a9accd4ac900e171908114a9164c722314b90fb2ca422b99603158f9ce3e
CACHE STRING "sqlite3 source hash")
if(SQLITE3_VERSION MATCHES "^([0-9]+)(0([0-9])|([1-9][0-9]))(0([0-9])|([1-9][0-9]))[0-9][0-9]$")
@ -223,6 +223,14 @@ function(build_external target)
endforeach()
string(REPLACE ___TARGET___ ${target} arg_BUILD_BYPRODUCTS "${arg_BUILD_BYPRODUCTS}")
set(externalproject_extra)
if(NOT CMAKE_VERSION VERSION_LESS 3.24)
# Default in cmake 3.24+ is to not extract timestamps for ExternalProject, which breaks pretty
# much every autotools package (which thinks it must reconfigure) because timestamps got
# updated).
list(APPEND externalproject_extra DOWNLOAD_EXTRACT_TIMESTAMP ON)
endif()
string(TOUPPER "${target}" prefix)
expand_urls(urls ${${prefix}_SOURCE} ${${prefix}_MIRROR})
ExternalProject_Add("${target}${arg_TARGET_SUFFIX}_external"
@ -237,6 +245,7 @@ function(build_external target)
BUILD_COMMAND ${arg_BUILD_COMMAND}
INSTALL_COMMAND ${arg_INSTALL_COMMAND}
BUILD_BYPRODUCTS ${arg_BUILD_BYPRODUCTS}
${externalproject_extra}
)
endfunction()

7
contrib/epee/include/epee/console_handler.h
View File

@ -338,11 +338,9 @@ eof:
color_prompt += "\001\033[0m\002";
m_stdin_reader.get_readline_buffer().set_prompt(color_prompt);
#else
epee::set_console_color(epee::console_color_yellow, true);
std::cout << prompt;
if (' ' != prompt.back())
std::cout << ' ';
epee::reset_console_color();
std::cout.flush();
#endif
}
@ -370,26 +368,22 @@ eof:
break;
if (m_stdin_reader.eos())
{
MGINFO("EOF on stdin, exiting");
std::cout << std::endl;
break;
}
if (m_cancel)
{
MDEBUG("Input cancelled");
cmd_handler(std::nullopt);
m_cancel = false;
continue;
}
if (!get_line_ret)
{
MERROR("Failed to read line.");
}
string_tools::trim(command);
LOG_PRINT_L2("Read command: " << command);
if (command.empty())
{
continue;
@ -405,7 +399,6 @@ eof:
}
catch (const std::exception &ex)
{
LOG_ERROR("Exception at [console_handler], what=" << ex.what());
}
}
if (exit_handler)

122
contrib/epee/include/epee/misc_log_ex.h
View File

@ -31,84 +31,12 @@
#ifdef __cplusplus
#include <string>
#include "easylogging++.h"
#include <sstream>
#include <iostream>
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "default"
#define MAX_LOG_FILE_SIZE 104850000 // 100 MB - 7600 bytes
#define MAX_LOG_FILES 50
#define CLOG_ENABLED(level, cat) ELPP->vRegistry()->allowed(el::Level::level, cat)
#define LOG_ENABLED(level) CLOG_ENABLED(level, OXEN_DEFAULT_LOG_CATEGORY)
#define MCLOG_TYPE(level, cat, type, x) do { \
if (ELPP->vRegistry()->allowed(level, cat)) { \
el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \
} \
} while (0)
#define MCLOG(level, cat, x) MCLOG_TYPE(level, cat, el::base::DispatchAction::NormalLog, x)
#define MCLOG_FILE(level, cat, x) MCLOG_TYPE(level, cat, el::base::DispatchAction::FileOnlyLog, x)
#define MCFATAL(cat,x) MCLOG(el::Level::Fatal,cat, x)
#define MCERROR(cat,x) MCLOG(el::Level::Error,cat, x)
#define MCWARNING(cat,x) MCLOG(el::Level::Warning,cat, x)
#define MCINFO(cat,x) MCLOG(el::Level::Info,cat, x)
#define MCDEBUG(cat,x) MCLOG(el::Level::Debug,cat, x)
#define MCTRACE(cat,x) MCLOG(el::Level::Trace,cat, x)
#define MCLOG_COLOR(level,cat,color,x) MCLOG(level,cat,"\033[1;" color "m" << x << "\033[0m")
#define MCLOG_RED(level,cat,x) MCLOG_COLOR(level,cat,"31",x)
#define MCLOG_GREEN(level,cat,x) MCLOG_COLOR(level,cat,"32",x)
#define MCLOG_YELLOW(level,cat,x) MCLOG_COLOR(level,cat,"33",x)
#define MCLOG_BLUE(level,cat,x) MCLOG_COLOR(level,cat,"34",x)
#define MCLOG_MAGENTA(level,cat,x) MCLOG_COLOR(level,cat,"35",x)
#define MCLOG_CYAN(level,cat,x) MCLOG_COLOR(level,cat,"36",x)
#define MLOG_RED(level,x) MCLOG_RED(level,OXEN_DEFAULT_LOG_CATEGORY,x)
#define MLOG_GREEN(level,x) MCLOG_GREEN(level,OXEN_DEFAULT_LOG_CATEGORY,x)
#define MLOG_YELLOW(level,x) MCLOG_YELLOW(level,OXEN_DEFAULT_LOG_CATEGORY,x)
#define MLOG_BLUE(level,x) MCLOG_BLUE(level,OXEN_DEFAULT_LOG_CATEGORY,x)
#define MLOG_MAGENTA(level,x) MCLOG_MAGENTA(level,OXEN_DEFAULT_LOG_CATEGORY,x)
#define MLOG_CYAN(level,x) MCLOG_CYAN(level,OXEN_DEFAULT_LOG_CATEGORY,x)
#define MFATAL(x) MCFATAL(OXEN_DEFAULT_LOG_CATEGORY,x)
#define MERROR(x) MCERROR(OXEN_DEFAULT_LOG_CATEGORY,x)
#define MWARNING(x) MCWARNING(OXEN_DEFAULT_LOG_CATEGORY,x)
#define MINFO(x) MCINFO(OXEN_DEFAULT_LOG_CATEGORY,x)
#define MDEBUG(x) MCDEBUG(OXEN_DEFAULT_LOG_CATEGORY,x)
#define MTRACE(x) MCTRACE(OXEN_DEFAULT_LOG_CATEGORY,x)
#define MLOG(level,x) MCLOG(level,OXEN_DEFAULT_LOG_CATEGORY,x)
#define MGINFO(x) MCINFO("global",x)
#define MGINFO_RED(x) MCLOG_RED(el::Level::Info, "global",x)
#define MGINFO_GREEN(x) MCLOG_GREEN(el::Level::Info, "global",x)
#define MGINFO_YELLOW(x) MCLOG_YELLOW(el::Level::Info, "global",x)
#define MGINFO_BLUE(x) MCLOG_BLUE(el::Level::Info, "global",x)
#define MGINFO_MAGENTA(x) MCLOG_MAGENTA(el::Level::Info, "global",x)
#define MGINFO_CYAN(x) MCLOG_CYAN(el::Level::Info, "global",x)
#define IFLOG(level, cat, type, init, x) \
do { \
if (ELPP->vRegistry()->allowed(level, cat)) { \
init; \
el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \
} \
} while(0)
#define MIDEBUG(init, x) IFLOG(el::Level::Debug, OXEN_DEFAULT_LOG_CATEGORY, el::base::DispatchAction::NormalLog, init, x)
#define LOG_ERROR(x) MERROR(x)
#define LOG_PRINT_L0(x) MWARNING(x)
#define LOG_PRINT_L1(x) MINFO(x)
#define LOG_PRINT_L2(x) MDEBUG(x)
#define LOG_PRINT_L3(x) MTRACE(x)
#define LOG_PRINT_L4(x) MTRACE(x)
#define MLOG_SET_THREAD_NAME(x) el::Helpers::setThreadName(x)
#ifndef LOCAL_ASSERT
#include <assert.h>
#if (defined _MSC_VER)
@ -119,13 +47,6 @@
#endif
std::string mlog_get_default_log_path(const char *default_filename);
void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size = MAX_LOG_FILE_SIZE, const std::size_t max_log_files = MAX_LOG_FILES);
void mlog_set_categories(const char *categories);
std::string mlog_get_categories();
void mlog_set_log_level(int level);
void mlog_set_log(const char *log);
namespace epee
{
namespace debug
@ -146,12 +67,10 @@ namespace debug
catch(const std::exception& ex) \
{ \
(void)(ex); \
LOG_ERROR("Exception at [" << location << "], what=" << ex.what()); \
return return_val; \
}\
catch(...)\
{\
LOG_ERROR("Exception at [" << location << "], generic exception \"...\"");\
return return_val; \
}
@ -161,39 +80,15 @@ namespace debug
#define CATCH_ENTRY_L3(lacation, return_val) CATCH_ENTRY(lacation, return_val)
#define CATCH_ENTRY_L4(lacation, return_val) CATCH_ENTRY(lacation, return_val)
#define ASSERT_MES_AND_THROW(message) {LOG_ERROR(message); std::stringstream ss; ss << message; throw std::runtime_error(ss.str());}
#define ASSERT_MES_AND_THROW(message) {std::stringstream ss; ss << message; throw std::runtime_error(ss.str());}
#define CHECK_AND_ASSERT_THROW_MES(expr, message) do {if(!(expr)) ASSERT_MES_AND_THROW(message);} while(0)
#ifndef CHECK_AND_ASSERT
#define CHECK_AND_ASSERT(expr, fail_ret_val) do{if(!(expr)){LOCAL_ASSERT(expr); return fail_ret_val;};}while(0)
#endif
#ifndef CHECK_AND_ASSERT_MES
#define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message) do{if(!(expr)) {LOG_ERROR(message); return fail_ret_val;};}while(0)
#endif
#ifndef CHECK_AND_NO_ASSERT_MES_L
#define CHECK_AND_NO_ASSERT_MES_L(expr, fail_ret_val, l, message) do{if(!(expr)) {LOG_PRINT_L##l(message); /*LOCAL_ASSERT(expr);*/ return fail_ret_val;};}while(0)
#endif
#ifndef CHECK_AND_NO_ASSERT_MES
#define CHECK_AND_NO_ASSERT_MES(expr, fail_ret_val, message) CHECK_AND_NO_ASSERT_MES_L(expr, fail_ret_val, 0, message)
#endif
#ifndef CHECK_AND_NO_ASSERT_MES_L1
#define CHECK_AND_NO_ASSERT_MES_L1(expr, fail_ret_val, message) CHECK_AND_NO_ASSERT_MES_L(expr, fail_ret_val, 1, message)
#endif
#ifndef CHECK_AND_ASSERT_MES_NO_RET
#define CHECK_AND_ASSERT_MES_NO_RET(expr, message) do{if(!(expr)) {LOG_ERROR(message); return;};}while(0)
#endif
#ifndef CHECK_AND_ASSERT_MES2
#define CHECK_AND_ASSERT_MES2(expr, message) do{if(!(expr)) {LOG_ERROR(message); };}while(0)
#define CHECK_AND_ASSERT_MES(expr, fail_ret_val, message) do{if(!(expr)) {return fail_ret_val;};}while(0)
#endif
enum console_colors
@ -209,9 +104,6 @@ enum console_colors
};
bool is_stdout_a_tty();
void set_console_color(int color, bool bright);
void reset_console_color();
}
extern "C"
@ -225,12 +117,6 @@ extern "C"
#define ATTRIBUTE_PRINTF
#endif
bool merror(const char *category, const char *format, ...) ATTRIBUTE_PRINTF;
bool mwarning(const char *category, const char *format, ...) ATTRIBUTE_PRINTF;
bool minfo(const char *category, const char *format, ...) ATTRIBUTE_PRINTF;
bool mdebug(const char *category, const char *format, ...) ATTRIBUTE_PRINTF;
bool mtrace(const char *category, const char *format, ...) ATTRIBUTE_PRINTF;
#ifdef __cplusplus
}

2
contrib/epee/include/epee/mlocker.h
View File

@ -86,5 +86,5 @@ namespace epee
template <class T, size_t N>
using mlocked_arr = mlocked<std::array<T, N>>;
template <typename T> constexpr bool is_byte_spannable<mlocked<T>> = is_byte_spannable<T>;
template <typename T> inline constexpr bool is_byte_spannable<mlocked<T>> = is_byte_spannable<T>;
}

156
contrib/epee/include/epee/net/abstract_tcp_server2.inl
View File

@ -51,9 +51,6 @@
#include <functional>
#include <random>
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "net"
#define AGGRESSIVE_TIMEOUT_THRESHOLD 120 // sockets
#define NEW_CONNECTION_TIMEOUT_LOCAL 1200000 // 2 minutes
#define NEW_CONNECTION_TIMEOUT_REMOTE 10000 // 10 seconds
@ -103,7 +100,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
m_local(false),
m_ready_to_close(false)
{
MDEBUG("test, connection constructor set m_connection_type="<<m_connection_type);
}
PRAGMA_WARNING_DISABLE_VS(4355)
@ -113,11 +109,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
{
if(!m_was_shutdown)
{
MTRACE("[sock " << socket().native_handle() << "] Socket destroyed without shutdown.");
shutdown();
}
MTRACE("[sock " << socket().native_handle() << "] Socket destroyed");
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
@ -141,9 +135,10 @@ PRAGMA_WARNING_DISABLE_VS(4355)
boost::system::error_code ec;
auto remote_ep = socket().remote_endpoint(ec);
CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get remote endpoint: " << ec.message() << ':' << ec.value());
CHECK_AND_NO_ASSERT_MES(remote_ep.address().is_v4() || remote_ep.address().is_v6(), false, "only IPv4 and IPv6 supported here");
if (ec)
throw std::runtime_error(std::string("Failed to get remote endpoint: ") + ec.message() + ":" + std::to_string(ec.value()));
if (!remote_ep.address().is_v4() && !remote_ep.address().is_v6())
throw std::runtime_error("Only IPv4 and IPv6 are supported here");
if (remote_ep.address().is_v4())
{
const unsigned long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong());
@ -176,17 +171,8 @@ PRAGMA_WARNING_DISABLE_VS(4355)
context = t_connection_context{};
context.set_details(random_uuid, std::move(real_remote), is_income);
boost::system::error_code ec;
auto local_ep = socket().local_endpoint(ec);
CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get local endpoint: " << ec.message() << ':' << ec.value());
MTRACE("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) <<
" to " << local_ep.address().to_string() << ':' << local_ep.port() <<
", total sockets objects " << get_state().sock_count);
if(static_cast<shared_state&>(get_state()).pfilter && !static_cast<shared_state&>(get_state()).pfilter->is_remote_host_allowed(context.m_remote_address))
{
MDEBUG("[sock " << socket().native_handle() << "] host denied " << context.m_remote_address.host_str() << ", shutdowning connection");
close();
return false;
}
@ -211,7 +197,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
boost::asio::detail::socket_option::integer< IPPROTO_IP, IP_TOS >
optionTos( tos );
socket().set_option( optionTos );
//MDEBUG("Set ToS flag to " << tos);
#endif
boost::asio::ip::tcp::no_delay noDelayOption(false);
@ -226,7 +211,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
bool connection<t_protocol_handler>::request_callback()
{
TRY_ENTRY();
MDEBUG("[" << print_connection_context_short(context) << "] request_callback");
// Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted
auto self = safe_shared_from_this();
if(!self)
@ -252,9 +236,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
auto self = safe_shared_from_this();
if(!self)
return false;
//MTRACE("[sock " << socket().native_handle() << "] add_ref, m_peer_number=" << mI->m_peer_number);
std::lock_guard lock{self->m_self_refs_lock};
//MTRACE("[sock " << socket().native_handle() << "] add_ref 2, m_peer_number=" << mI->m_peer_number);
if(m_was_shutdown)
return false;
++m_reference_count;
@ -268,7 +250,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
{
TRY_ENTRY();
std::shared_ptr<connection<t_protocol_handler> > back_connection_copy;
LOG_TRACE_CC(context, "[sock " << socket().native_handle() << "] release");
std::lock_guard lock{m_self_refs_lock};
CHECK_AND_ASSERT_MES(m_reference_count, false, "[sock " << socket().native_handle() << "] m_reference_count already at 0 at connection<t_protocol_handler>::release() call");
// is this the last reference?
@ -284,7 +265,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
void connection<t_protocol_handler>::call_back_starter()
{
TRY_ENTRY();
MDEBUG("[" << print_connection_context_short(context) << "] fired_callback");
m_protocol_handler.handle_qued_callback();
CATCH_ENTRY_L0("connection<t_protocol_handler>::call_back_starter()", void());
}
@ -306,9 +286,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
address = endpoint.address().to_string();
port = boost::lexical_cast<std::string>(endpoint.port());
}
MDEBUG(" connection type " << to_string( m_connection_type ) << " "
<< socket().local_endpoint().address().to_string() << ":" << socket().local_endpoint().port()
<< " <--> " << context.m_remote_address.str() << " (via " << address << ":" << port << ")");
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
@ -316,7 +293,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
std::size_t bytes_transferred)
{
TRY_ENTRY();
//MINFO("[sock " << socket().native_handle() << "] Async read calledback.");
if (!e)
{
@ -354,7 +330,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
} while(delay > 0);
} // any form of sleeping
//MINFO("[sock " << socket().native_handle() << "] RECV " << bytes_transferred);
logger_handle_net_read(bytes_transferred);
context.m_last_recv = std::chrono::steady_clock::now();
context.m_recv_cnt += bytes_transferred;
@ -362,7 +337,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
bool recv_res = m_protocol_handler.handle_recv(buffer_.data(), bytes_transferred);
if(!recv_res)
{
//MINFO("[sock " << socket().native_handle() << "] protocol_want_close");
//some error in protocol, protocol handler ask to close connection
m_want_close_connection = true;
bool do_shutdown = false;
@ -381,19 +355,15 @@ PRAGMA_WARNING_DISABLE_VS(4355)
boost::bind(&connection<t_protocol_handler>::handle_read, connection<t_protocol_handler>::shared_from_this(),
boost::asio::placeholders::error,
boost::asio::placeholders::bytes_transferred)));
//MINFO("[sock " << socket().native_handle() << "]Async read requested.");
}
}else
{
MTRACE("[sock " << socket().native_handle() << "] Some not success at read: " << e.message() << ':' << e.value());
if(e.value() != 2)
{
MTRACE("[sock " << socket().native_handle() << "] Some problems at read: " << e.message() << ':' << e.value());
shutdown();
}
else
{
MTRACE("[sock " << socket().native_handle() << "] peer closed connection");
bool do_shutdown = false;
{
std::lock_guard lock{m_send_que_lock};
@ -461,21 +431,14 @@ PRAGMA_WARNING_DISABLE_VS(4355)
{ // LOCK: chunking
std::lock_guard send_guard{m_chunking_lock};
MDEBUG("do_send() will SPLIT into small chunks, from packet="<<message.size()<<" B for ptr="<<(void*)message.ptr.get());
while (!message.view.empty()) {
bool ok = do_send_chunk(message.extract_prefix(chunksize_good));
if (!ok) {
MDEBUG("do_send() DONE ***FAILED***");
MDEBUG("do_send() SEND was aborted in middle of big package - this is probably harmless (e.g. peer closed connection)");
return false; // partial failure in sending
}
}
MDEBUG("do_send() DONE SPLIT send");
MDEBUG("do_send() m_connection_type = " << m_connection_type);
return true; // done - e.g. queued - all the chunks of current do_send call
} // LOCK: chunking
} // a big block (to be chunked) - all chunks
@ -506,7 +469,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
context.m_current_speed_up = current_speed_up;
context.m_max_speed_up = std::max(context.m_max_speed_up, current_speed_up);
//MINFO("[sock " << socket().native_handle() << "] SEND " << cb);
context.m_last_send = std::chrono::steady_clock::now();
context.m_send_cnt += chunk.size();
//some data should be wrote to stream
@ -522,11 +484,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
{
retry++;
/* if ( ::cryptonote::core::get_is_stopping() ) { // TODO re-add fast stop
MDEBUG("ABORT queue wait due to stopping");
return false; // aborted
}*/
using engine = std::mt19937;
static bool initialized = false;
static engine rng;
@ -541,14 +498,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
}
int ms = std::uniform_int_distribution<>{250, 299}(rng);
MDEBUG("Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<chunk.size()); // XXX debug sleep
queue_lock.unlock();
std::this_thread::sleep_for(std::chrono::milliseconds{ms});
queue_lock.lock();
MDEBUG("sleep for queue: " << ms);
if (retry > retry_limit) {
MWARNING("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
shutdown();
return false;
}
@ -559,22 +513,18 @@ PRAGMA_WARNING_DISABLE_VS(4355)
if(m_send_que.size() > 1)
{ // active operation should be in progress, nothing to do, just wait last operation callback
auto size_now = m_send_que.back().size();
MDEBUG("do_send_chunk() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size());
//do_send_handler_delayed( ptr , size_now ); // (((H))) // empty function
LOG_TRACE_CC(context, "[sock " << socket().native_handle() << "] Async send requested " << m_send_que.front().size());
}
else
{ // no active operation
if(m_send_que.size()!=1)
{
MERROR("Looks like no active operations, but send que size != 1!!");
return false;
}
auto size_now = m_send_que.front().size();
MDEBUG("do_send_chunk() NOW SENSD: packet="<<size_now<<" B");
if (speed_limit_is_enabled())
do_send_handler_write( m_send_que.back().data(), m_send_que.back().size() ); // (((H)))
@ -586,9 +536,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2)
)
);
//MTRACE("(chunk): " << size_now);
//logger_handle_net_write(size_now);
//MINFO("[sock " << socket().native_handle() << "] Async send requested " << m_send_que.front().size());
}
//do_send_handler_stop( ptr , cb ); // empty function
@ -629,10 +576,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
std::lock_guard lock{hosts_mutex};
static std::map<std::string, unsigned int> hosts;
unsigned int &val = hosts[host];
if (delta > 0)
MTRACE("New connection from host " << host << ": " << val);
else if (delta < 0)
MTRACE("Closed connection from host " << host << ": " << val);
CHECK_AND_ASSERT_THROW_MES(delta >= 0 || val >= (unsigned)-delta, "Count would go negative");
CHECK_AND_ASSERT_THROW_MES(delta <= 0 || val <= std::numeric_limits<unsigned int>::max() - (unsigned)delta, "Count would wrap");
val += delta;
@ -644,19 +587,15 @@ PRAGMA_WARNING_DISABLE_VS(4355)
{
if (ms < 0s)
{
MWARNING("Ignoring negative timeout " << ms.count());
return;
}
MTRACE((add ? "Adding" : "Setting") << " " << ms.count() << "ms expiry");
auto self = safe_shared_from_this();
if(!self)
{
MERROR("Resetting timer on a dead object");
return;
}
if (m_was_shutdown)
{
MERROR("Setting timer on a shut down object");
return;
}
if (add)
@ -670,7 +609,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
{
if(ec == boost::asio::error::operation_aborted)
return;
MDEBUG(context << "connection timeout, closing");
self->close();
});
}
@ -703,7 +641,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
auto self = safe_shared_from_this();
if(!self)
return false;
//MINFO("[sock " << socket().native_handle() << "] Que Shutdown called.");
m_timer.cancel();
size_t send_que_size = 0;
{
@ -739,11 +676,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
void connection<t_protocol_handler>::handle_write(const boost::system::error_code& e, size_t cb)
{
TRY_ENTRY();
LOG_TRACE_CC(context, "[sock " << socket().native_handle() << "] Async send calledback " << cb);
if (e)
{
MDEBUG("[sock " << socket().native_handle() << "] Some problems at write: " << e.message() << ':' << e.value());
shutdown();
return;
}
@ -758,7 +693,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
std::unique_lock lock{m_send_que_lock};
if(m_send_que.empty())
{
MERROR("[sock " << socket().native_handle() << "] m_send_que.size() == 0 at handle_write!");
return;
}
@ -774,7 +708,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
//have more data to send
reset_timer(get_default_timeout(), false);
auto size_now = m_send_que.front().size();
MDEBUG("handle_write() NOW SENDS: packet="<<size_now<<" B" <<", from queue size="<<m_send_que.size());
if (speed_limit_is_enabled())
do_send_handler_write_from_queue(e, m_send_que.front().size() , m_send_que.size()); // (((H)))
CHECK_AND_ASSERT_MES( size_now == m_send_que.front().size(), void(), "Unexpected queue size");
@ -784,7 +717,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2)
)
);
//MTRACE("(normal)" << size_now);
}
lock.unlock();
@ -800,10 +732,8 @@ PRAGMA_WARNING_DISABLE_VS(4355)
void connection<t_protocol_handler>::setRpcStation()
{
m_connection_type = e_connection_type_RPC;
MDEBUG("set m_connection_type = RPC ");
}
template<class t_protocol_handler>
bool connection<t_protocol_handler>::speed_limit_is_enabled() const {
return m_connection_type != e_connection_type_RPC ;
@ -893,7 +823,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
acceptor_.listen();
boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint();
m_port = binded_endpoint.port();
MDEBUG("start accept (IPv4)");
new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type));
acceptor_.async_accept(new_connection_->socket(),
boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4, this,
@ -906,7 +835,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
if (ipv4_failed != "")
{
MERROR("Failed to bind IPv4: " << ipv4_failed);
if (require_ipv4)
{
throw std::runtime_error("Failed to bind IPv4 (set to required)");
@ -930,7 +858,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
acceptor_ipv6.listen();
boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_ipv6.local_endpoint();
m_port_ipv6 = binded_endpoint.port();
MDEBUG("start accept (IPv6)");
new_connection_ipv6.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type));
acceptor_ipv6.async_accept(new_connection_ipv6->socket(),
boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6, this,
@ -944,7 +871,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
if (use_ipv6 && ipv6_failed != "")
{
MERROR("Failed to bind IPv6: " << ipv6_failed);
if (ipv4_failed != "")
{
throw std::runtime_error("Failed to bind IPv4 and IPv6");
@ -955,12 +881,12 @@ PRAGMA_WARNING_DISABLE_VS(4355)
}
catch (const std::exception &e)
{
MFATAL("Error starting server: " << e.what());
throw std::runtime_error(std::string("Error starting server: {}") + e.what());
return false;
}
catch (...)
{
MFATAL("Error starting server");
throw std::runtime_error("Error starting server");
return false;
}
}
@ -975,12 +901,10 @@ DISABLE_GCC_WARNING(maybe-uninitialized)
uint32_t p_ipv6 = 0;
if (port.size() && !string_tools::get_xtype_from_string(p, port)) {
MERROR("Failed to convert port no = " << port);
return false;
}
if (port_ipv6.size() && !string_tools::get_xtype_from_string(p_ipv6, port_ipv6)) {
MERROR("Failed to convert port no = " << port_ipv6);
return false;
}
return this->init_server(p, address, p_ipv6, address_ipv6, use_ipv6, require_ipv4);
@ -991,11 +915,6 @@ POP_WARNINGS
bool boosted_tcp_server<t_protocol_handler>::worker_thread()
{
TRY_ENTRY();
uint32_t local_thr_index = m_thread_index++;
std::string thread_name = std::string("[") + m_thread_name_prefix;
thread_name += std::to_string(local_thr_index) + "]";
MLOG_SET_THREAD_NAME(thread_name);
// MDEBUG("Thread name: " << m_thread_name_prefix);
while(!m_stop_signal_sent)
{
try
@ -1003,16 +922,10 @@ POP_WARNINGS
io_service_.run();
return true;
}
catch(const std::exception& ex)
{
MERROR("Exception at server worker thread, what=" << ex.what());
}
catch(...)
{
MERROR("Exception at server worker thread, unknown execption");
}
}
//MINFO("Worker thread finished");
return true;
CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::worker_thread", false);
}
@ -1024,7 +937,6 @@ POP_WARNINGS
auto it = server_type_map.find(m_thread_name_prefix);
if (it==server_type_map.end()) throw std::runtime_error("Unknown prefix/server type:" + std::string(prefix_name));
auto connection_type = it->second; // the value of type
MINFO("Set server type to: " << connection_type << " from name: " << m_thread_name_prefix << ", prefix_name = " << prefix_name);
}
//---------------------------------------------------------------------------------
template<class t_protocol_handler>
@ -1040,7 +952,6 @@ POP_WARNINGS
TRY_ENTRY();
m_threads_count = threads_count;
m_main_thread_id = std::this_thread::get_id();
MLOG_SET_THREAD_NAME("[SRV_MAIN]");
while(!m_stop_signal_sent)
{
@ -1050,36 +961,28 @@ POP_WARNINGS
for (std::size_t i = 0; i < threads_count; ++i)
{
m_threads.emplace_back([this] { worker_thread(); });
MDEBUG("Run server thread name: " << m_thread_name_prefix);
}
}
// Wait for all threads in the pool to exit.
if (wait)
{
MDEBUG("JOINING all threads");
for (auto& th: m_threads)
th.join();
MDEBUG("JOINING all threads - almost");
m_threads.clear();
MDEBUG("JOINING all threads - DONE");
}
else {
MDEBUG("Reiniting OK.");
return true;
}
if(wait && !m_stop_signal_sent)
{
//some problems with the listening socket ?..
MDEBUG("Net service stopped without stop request, restarting...");
if(!this->init_server(m_port, m_address, m_port_ipv6, m_address_ipv6, m_use_ipv6, m_require_ipv4))
{
MDEBUG("Reiniting service failed, exit.");
return false;
}else
{
MDEBUG("Reiniting OK.");
//}else
//{
}
}
}
@ -1145,8 +1048,6 @@ POP_WARNINGS
template<class t_protocol_handler>
void boosted_tcp_server<t_protocol_handler>::handle_accept(const boost::system::error_code& e, bool ipv6)
{
MDEBUG("handle_accept");
boost::asio::ip::tcp::acceptor* current_acceptor = &acceptor_;
connection_ptr* current_new_connection = &new_connection_;
auto accept_function_pointer = &boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4;
@ -1162,7 +1063,6 @@ POP_WARNINGS
if (!e)
{
if (m_connection_type == e_connection_type_RPC) {
MDEBUG("New server for RPC connections");
(*current_new_connection)->setRpcStation(); // hopefully this is not needed actually
}
connection_ptr conn(std::move((*current_new_connection)));
@ -1187,19 +1087,13 @@ POP_WARNINGS
conn->save_dbg_log();
return;
}
else
{
MERROR("Error in boosted_tcp_server<t_protocol_handler>::handle_accept: " << e);
}
}
catch (const std::exception &e)
{
MERROR("Exception in boosted_tcp_server<t_protocol_handler>::handle_accept: " << e.what());
}
// error path, if e or exception
assert(m_state != nullptr); // always set in constructor
MERROR("Some problems at accept: " << e.message() << ", connections_count = " << m_state->sock_count);
std::this_thread::sleep_for(100ms);
(*current_new_connection).reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type));
current_acceptor->async_accept((*current_new_connection)->socket(),
@ -1220,10 +1114,6 @@ POP_WARNINGS
return true;
}
}
else
{
MWARNING(out << " was not added, socket/io_service mismatch");
}
return false;
}
//---------------------------------------------------------------------------------
@ -1240,7 +1130,6 @@ POP_WARNINGS
sock_.bind(local_endpoint, ec);
if (ec)
{
MERROR("Error binding to " << bind_ip << ": " << ec.message());
if (sock_.is_open())
sock_.close();
return CONNECT_FAILURE;
@ -1284,7 +1173,6 @@ POP_WARNINGS
{
//timeout
sock_.close();
MTRACE("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")");
return CONNECT_FAILURE;
}
}
@ -1292,14 +1180,11 @@ POP_WARNINGS
if (ec || !sock_.is_open())
{
MTRACE("Some problems at connect, message: " << ec.message());
if (sock_.is_open())
sock_.close();
return CONNECT_FAILURE;
}
MTRACE("Connected success to " << adr << ':' << port);
return CONNECT_SUCCESS;
CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::try_connect", CONNECT_FAILURE);
@ -1313,7 +1198,6 @@ POP_WARNINGS
connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type) );
connections_mutex.lock();
connections_.insert(new_connection_l);
MDEBUG("connections_ size now " << connections_.size());
connections_mutex.unlock();
auto scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ std::lock_guard lock{connections_mutex}; connections_.erase(new_connection_l); });
boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket();
@ -1350,13 +1234,11 @@ POP_WARNINGS
{
if (!m_use_ipv6)
{
MERROR("Failed to resolve " << adr);
return false;
}
else
{
try_ipv6 = true;
MINFO("Resolving address as IPv4 failed, trying IPv6");
}
}
else
@ -1372,7 +1254,6 @@ POP_WARNINGS
if(iterator == end)
{
MERROR("Failed to resolve " << adr);
return false;
}
else
@ -1390,9 +1271,6 @@ POP_WARNINGS
}
MDEBUG("Trying to connect to " << adr << ":" << port << ", bind_ip = " << bind_ip_to_use);
//boost::asio::ip::tcp::endpoint remote_endpoint(boost::asio::ip::address::from_string(addr.c_str()), port);
boost::asio::ip::tcp::endpoint remote_endpoint(*iterator);
auto try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip_to_use, conn_timeout);
@ -1412,7 +1290,6 @@ POP_WARNINGS
else
{
assert(m_state != nullptr); // always set in constructor
MERROR("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection, connections_count = " << m_state->sock_count);
}
new_connection_l->save_dbg_log();
@ -1429,7 +1306,6 @@ POP_WARNINGS
connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type) );
connections_mutex.lock();
connections_.insert(new_connection_l);
MDEBUG("connections_ size now " << connections_.size());
connections_mutex.unlock();
auto scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ std::lock_guard lock{connections_mutex}; connections_.erase(new_connection_l); });
boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket();
@ -1465,13 +1341,8 @@ POP_WARNINGS
{
if (!try_ipv6)
{
MERROR("Failed to resolve " << adr);
return false;
}
else
{
MINFO("Resolving address as IPv4 failed, trying IPv6");
}
}
if (try_ipv6)
@ -1482,7 +1353,6 @@ POP_WARNINGS
if(iterator == end)
{
MERROR("Failed to resolve " << adr);
return false;
}
}
@ -1498,7 +1368,6 @@ POP_WARNINGS
sock_.bind(local_endpoint, ec);
if (ec)
{
MERROR("Error binding to " << bind_ip << ": " << ec.message());
if (sock_.is_open())
sock_.close();
return false;
@ -1512,7 +1381,6 @@ POP_WARNINGS
{
if(error != boost::asio::error::operation_aborted)
{
MTRACE("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")");
new_connection_l->socket().close();
}
});
@ -1520,8 +1388,6 @@ POP_WARNINGS
sock_.async_connect(remote_endpoint, [=](const boost::system::error_code& ec_)
{
t_connection_context conn_context{};
boost::system::error_code ignored_ec;
boost::asio::ip::tcp::socket::endpoint_type lep = new_connection_l->socket().local_endpoint(ignored_ec);
if(!ec_)
{//success
if(!sh_deadline->cancel())
@ -1529,9 +1395,6 @@ POP_WARNINGS
cb(conn_context, boost::asio::error::operation_aborted);//this mean that deadline timer already queued callback with cancel operation, rare situation
}else
{
MTRACE("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port <<
" from " << lep.address().to_string() << ':' << lep.port());
// start adds the connection to the config object's list, so we don't need to have it locally anymore
connections_mutex.lock();
connections_.erase(new_connection_l);
@ -1544,14 +1407,11 @@ POP_WARNINGS
}
else
{
MTRACE("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port);
cb(conn_context, boost::asio::error::fault);
}
}
}else
{
MTRACE("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port <<
" from " << lep.address().to_string() << ':' << lep.port() << ": " << ec_.message() << ':' << ec_.value());
cb(conn_context, ec_);
}
});

4
contrib/epee/include/epee/net/buffer.h
View File

@ -32,10 +32,6 @@
#include "../misc_log_ex.h"
#include "../span.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "net.buffer"
//#define NET_BUFFER_LOG(x) MDEBUG(x)
#define NET_BUFFER_LOG(x) ((void)0)
namespace epee

7
contrib/epee/include/epee/net/levin_protocol_handler.h
View File

@ -33,9 +33,6 @@
#include "levin_base.h"
#include "../int-util.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "net"
namespace epee
{
namespace levin
@ -91,7 +88,6 @@ namespace levin
{
if(!m_config.m_pcommands_handler)
{
LOG_ERROR_CC(m_conn_context, "Command handler not set!");
return false;
}
m_cach_in_buffer.append((const char*)ptr, cb);
@ -106,7 +102,6 @@ namespace levin
{
if(m_cach_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cach_in_buffer.data()) != SWAP64LE(LEVIN_SIGNATURE))
{
LOG_ERROR_CC(m_conn_context, "Signature mismatch on accepted connection");
return false;
}
is_continue = false;
@ -126,7 +121,6 @@ namespace levin
#endif
if(LEVIN_SIGNATURE != phead.m_signature)
{
LOG_ERROR_CC(m_conn_context, "Signature mismatch on accepted connection");
return false;
}
m_current_head = phead;
@ -169,7 +163,6 @@ namespace levin
m_state = conn_state_reading_head;
break;
default:
LOG_ERROR_CC(m_conn_context, "Undefined state in levin_server_impl::connection_handler, m_state=" << m_state);
return false;
}
}

52
contrib/epee/include/epee/net/levin_protocol_handler_async.h
View File

@ -32,6 +32,7 @@
#include <atomic>
#include <memory>
#include <thread>
#include "levin_base.h"
#include "buffer.h"
@ -41,9 +42,6 @@
#include <random>
#include <chrono>
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "net"
#ifndef MIN_BYTES_WANTED
#define MIN_BYTES_WANTED 512
#endif
@ -135,11 +133,6 @@ class async_protocol_handler
if(!m_pservice_endpoint->do_send(shared_sv{std::move(data)}))
return false;
MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb
<< ", flags" << head.m_flags
<< ", r?=" << head.m_have_to_return_data
<<", cmd = " << head.m_command
<< ", ver=" << head.m_protocol_version);
return true;
}
@ -194,13 +187,11 @@ public:
{
if(m_con.start_outer_call())
{
MDEBUG(con.get_context_ref() << "anvoke_handler, timeout: " << timeout.count());
m_timer.expires_from_now(std::chrono::milliseconds(timeout));
m_timer.async_wait([&con, command, cb, timeout](const boost::system::error_code& ec)
{
if(ec == boost::asio::error::operation_aborted)
return;
MINFO(con.get_context_ref() << "Timeout on invoke operation happened, command: " << command << " timeout: " << timeout.count());
epee::span<const uint8_t> fake;
cb(LEVIN_ERROR_CONNECTION_TIMEDOUT, fake, con.get_context_ref());
con.close();
@ -263,7 +254,6 @@ public:
{
if(ec == boost::asio::error::operation_aborted)
return;
MINFO(con.get_context_ref() << "Timeout on invoke operation happened, command: " << command << " timeout: " << timeout.count());
epee::span<const uint8_t> fake;
cb(LEVIN_ERROR_CONNECTION_TIMEDOUT, fake, con.get_context_ref());
con.close();
@ -282,7 +272,6 @@ public:
std::lock_guard lock{m_invoke_response_handlers_lock};
if (m_protocol_released)
{
MERROR("Adding response handler to a released object");
return false;
}
std::shared_ptr<invoke_response_handler_base> handler(std::make_shared<anvoke_handler<callback_t>>(cb, timeout, con, command));
@ -325,20 +314,14 @@ public:
{
std::this_thread::sleep_for(100ms);
}
CHECK_AND_ASSERT_MES_NO_RET(0 == m_wait_count, "Failed to wait for operation completion. m_wait_count = " << m_wait_count);
MTRACE(m_connection_context << "~async_protocol_handler()");
}
catch (...) { /* ignore */ }
}
bool start_outer_call()
{
MTRACE(m_connection_context << "[levin_protocol] -->> start_outer_call");
if(!m_pservice_endpoint->add_ref())
{
MERROR(m_connection_context << "[levin_protocol] -->> start_outer_call failed");
return false;
}
m_wait_count++;
@ -346,7 +329,6 @@ public:
}
bool finish_outer_call()
{
MTRACE(m_connection_context << "[levin_protocol] <<-- finish_outer_call");
m_wait_count--;
m_pservice_endpoint->release();
return true;
@ -403,7 +385,6 @@ public:
if(!m_config.m_pcommands_handler)
{
MERROR(m_connection_context << "Commands handler not set!");
return false;
}
@ -414,9 +395,6 @@ public:
// flipped to subtraction; prevent overflow since m_max_packet_size is variable and public
if(cb > m_config.m_max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size())
{
MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size
<< ", packet received " << m_cache_in_buffer.size() + cb
<< ", connection will be closed.");
return false;
}
@ -439,7 +417,6 @@ public:
//async call scenario
std::shared_ptr<invoke_response_handler_base> response_handler = m_invoke_response_handlers.front();
response_handler->reset_timer();
MDEBUG(m_connection_context << "LEVIN_PACKET partial msg received. len=" << cb);
}
}
break;
@ -467,7 +444,6 @@ public:
if (m_fragment_buffer.size() < sizeof(bucket_head2))
{
MERROR(m_connection_context << "Fragmented data too small for levin header");
return false;
}
@ -479,12 +455,6 @@ public:
bool is_response = (m_oponent_protocol_ver == LEVIN_PROTOCOL_VER_1 && m_current_head.m_flags&LEVIN_PACKET_RESPONSE);
MDEBUG(m_connection_context << "LEVIN_PACKET_RECEIVED. [len=" << m_current_head.m_cb
<< ", flags" << m_current_head.m_flags
<< ", r?=" << m_current_head.m_have_to_return_data
<<", cmd = " << m_current_head.m_command
<< ", v=" << m_current_head.m_protocol_version);
if(is_response)
{//response to some invoke
@ -507,7 +477,6 @@ public:
//use sync call scenario
if(m_wait_count == 0 && m_close_called == 0)
{
MERROR(m_connection_context << "no active invoke when response came, wtf?");
return false;
}else
{
@ -533,12 +502,6 @@ public:
if(!m_pservice_endpoint->do_send(shared_sv{std::move(return_buff)}))
return false;
MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb
<< ", flags" << head.m_flags
<< ", r?=" << head.m_have_to_return_data
<<", cmd = " << head.m_command
<< ", ver=" << head.m_protocol_version);
}
else
m_config.m_pcommands_handler->notify(m_current_head.m_command, buff_to_invoke, m_connection_context);
@ -557,7 +520,6 @@ public:
{
if(m_cache_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cache_in_buffer.span(8).data()) != SWAP64LE(LEVIN_SIGNATURE))
{
MWARNING(m_connection_context << "Signature mismatch, connection will be closed");
return false;
}
is_continue = false;
@ -577,7 +539,6 @@ public:
#endif
if(LEVIN_SIGNATURE != phead.m_signature)
{
LOG_ERROR_CC(m_connection_context, "Signature mismatch, connection will be closed");
return false;
}
m_current_head = phead;
@ -587,15 +548,11 @@ public:
m_oponent_protocol_ver = m_current_head.m_protocol_version;
if(m_current_head.m_cb > m_config.m_max_packet_size)
{
LOG_ERROR_CC(m_connection_context, "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size
<< ", packet header received " << m_current_head.m_cb
<< ", connection will be closed.");
return false;
}
}
break;
default:
LOG_ERROR_CC(m_connection_context, "Undefined state in levin_server_impl::connection_handler, m_state=" << m_state);
return false;
}
}
@ -645,7 +602,6 @@ public:
if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true))
{
LOG_ERROR_CC(m_connection_context, "Failed to do_send");
err_code = LEVIN_ERROR_CONNECTION;
break;
}
@ -686,7 +642,6 @@ public:
if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true))
{
LOG_ERROR_CC(m_connection_context, "Failed to send request");
return LEVIN_ERROR_CONNECTION;
}
@ -702,7 +657,6 @@ public:
}
if (std::chrono::steady_clock::now() > start + m_config.m_invoke_timeout)
{
MWARNING(m_connection_context << "invoke timeout (" << m_config.m_invoke_timeout.count() << "), closing connection ");
close();
return LEVIN_ERROR_CONNECTION_TIMEDOUT;
}
@ -735,7 +689,6 @@ public:
if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, false))
{
LOG_ERROR_CC(m_connection_context, "Failed to send notify message");
return -1;
}
@ -759,11 +712,9 @@ public:
const std::size_t length = message.view.size();
if (!m_pservice_endpoint->do_send(std::move(message)))
{
LOG_ERROR_CC(m_connection_context, "Failed to send message, dropping it");
return -1;
}
MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << (length - sizeof(bucket_head2)) << ", r?=0]");
return 1;
}
//------------------------------------------------------------------------------------------
@ -809,7 +760,6 @@ void async_protocol_handler_config<t_connection_context>::delete_connections(siz
}
catch (const std::out_of_range &e)
{
MWARNING("Connection not found in m_connects, continuing");
}
--count;
}

31
contrib/epee/include/epee/net/net_utils_base.h
View File

@ -437,36 +437,15 @@ namespace net_utils
//some helpers
std::string print_connection_context(const connection_context_base& ctx);
std::string print_connection_context_short(const connection_context_base& ctx);
inline MAKE_LOGGABLE(connection_context_base, ct, os)
{
os << "[" << epee::net_utils::print_connection_context_short(ct) << "] ";
return os;
}
inline std::ostream& operator<<(std::ostream& os, const connection_context_base& ct)
{
os << "[" << epee::net_utils::print_connection_context_short(ct) << "] ";
return os;
}
#define LOG_ERROR_CC(ct, message) MERROR(ct << message)
#define LOG_WARNING_CC(ct, message) MWARNING(ct << message)
#define LOG_INFO_CC(ct, message) MINFO(ct << message)
#define LOG_DEBUG_CC(ct, message) MDEBUG(ct << message)
#define LOG_TRACE_CC(ct, message) MTRACE(ct << message)
#define LOG_CC(level, ct, message) MLOG(level, ct << message)
#define LOG_PRINT_CC_L0(ct, message) LOG_PRINT_L0(ct << message)
#define LOG_PRINT_CC_L1(ct, message) LOG_PRINT_L1(ct << message)
#define LOG_PRINT_CC_L2(ct, message) LOG_PRINT_L2(ct << message)
#define LOG_PRINT_CC_L3(ct, message) LOG_PRINT_L3(ct << message)
#define LOG_PRINT_CC_L4(ct, message) LOG_PRINT_L4(ct << message)
#define LOG_PRINT_CCONTEXT_L0(message) LOG_PRINT_CC_L0(context, message)
#define LOG_PRINT_CCONTEXT_L1(message) LOG_PRINT_CC_L1(context, message)
#define LOG_PRINT_CCONTEXT_L2(message) LOG_PRINT_CC_L2(context, message)
#define LOG_PRINT_CCONTEXT_L3(message) LOG_PRINT_CC_L3(context, message)
#define LOG_ERROR_CCONTEXT(message) LOG_ERROR_CC(context, message)
#define CHECK_AND_ASSERT_MES_CC(condition, return_val, err_message) CHECK_AND_ASSERT_MES(condition, return_val, "[" << epee::net_utils::print_connection_context_short(context) << "]" << err_message)
}

8
contrib/epee/include/epee/serialization/keyvalue_serialization.h
View File

@ -29,10 +29,6 @@
#include "../misc_log_ex.h"
#include "keyvalue_serialization_overloads.h"
#include "../storages/portable_storage.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "serialization"
namespace epee
{
/************************************************************************/
@ -66,8 +62,7 @@ public: \
bool Class::load(epee::serialization::portable_storage& st, epee::serialization::section* parent_section) \
{ \
try { return _load(st, parent_section); } \
catch (const std::exception& err) { LOG_ERROR("Deserialization exception: " << err.what()); } \
catch (...) { LOG_ERROR("Unknown deserialization exception"); } \
catch (...) {} \
return false; \
} \
template <bool is_store> \
@ -98,7 +93,6 @@ public: \
catch(const std::exception& err) \
{ \
(void)(err); \
LOG_ERROR("Exception on deserializing: " << err.what());\
return false; \
}\
}\

2
contrib/epee/include/epee/serialization/keyvalue_serialization_overloads.h
View File

@ -34,6 +34,7 @@
#include <deque>
#include <array>
#include <optional>
#include <cstring>
#include "../span.h"
#include "../storages/portable_storage_base.h"
@ -152,7 +153,6 @@ namespace epee
return true;
} catch (const std::out_of_range&) { // ignore silently
} catch (const std::exception& e) {
LOG_ERROR("Failed to deserialize stl container: " << e.what());
}
return false;
}

19
contrib/epee/include/epee/storages/levin_abstract_invoke2.h
View File

@ -30,9 +30,6 @@
#include "../span.h"
#include "../net/levin_base.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "net"
namespace epee
{
namespace net_utils
@ -51,13 +48,11 @@ namespace epee
int res = transport.invoke(command, buff_to_send, buff_to_recv);
if( res <=0 )
{
MERROR("Failed to invoke command " << command << " return code " << res);
return false;
}
serialization::portable_storage stg_ret;
if(!stg_ret.load_from_binary(buff_to_recv))
{
LOG_ERROR("Failed to load_from_binary on command " << command);
return false;
}
return result_struct.load(stg_ret);
@ -77,7 +72,6 @@ namespace epee
int res = transport.notify(command, buff_to_send);
if(res <=0 )
{
LOG_ERROR("Failed to notify command " << command << " return code " << res);
return false;
}
return true;
@ -95,13 +89,11 @@ namespace epee
int res = transport.invoke(command, buff_to_send, buff_to_recv, conn_id);
if( res <=0 )
{
LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res);
return false;
}
typename serialization::portable_storage stg_ret;
if(!stg_ret.load_from_binary(buff_to_recv))
{
LOG_ERROR("Failed to load_from_binary on command " << command);
return false;
}
return result_struct.load(stg_ret);
@ -119,20 +111,17 @@ namespace epee
t_result result_struct{};
if( code <=0 )
{
LOG_PRINT_L1("Failed to invoke command " << command << " return code " << code);
cb(code, std::move(result_struct), context);
return false;
}
serialization::portable_storage stg_ret;
if(!stg_ret.load_from_binary(buff))
{
LOG_ERROR("Failed to load_from_binary on command " << command);
cb(LEVIN_ERROR_FORMAT, std::move(result_struct), context);
return false;
}
if (!result_struct.load(stg_ret))
{
LOG_ERROR("Failed to load result struct on command " << command);
cb(LEVIN_ERROR_FORMAT, std::move(result_struct), context);
return false;
}
@ -141,7 +130,6 @@ namespace epee
}, inv_timeout);
if( res <=0 )
{
LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res);
return false;
}
return true;
@ -159,7 +147,6 @@ namespace epee
int res = transport.notify(command, epee::strspan<uint8_t>(buff_to_send), conn_id);
if(res <=0 )
{
MERROR("Failed to notify command " << command << " return code " << res);
return false;
}
return true;
@ -172,7 +159,6 @@ namespace epee
serialization::portable_storage strg;
if(!strg.load_from_binary(in_buff))
{
LOG_ERROR("Failed to load_from_binary in command " << command);
return -1;
}
t_in_type in_struct{};
@ -180,7 +166,6 @@ namespace epee
if (!in_struct.load(strg))
{
LOG_ERROR("Failed to load in_struct in command " << command);
return -1;
}
int res = cb(command, in_struct, out_struct, context);
@ -189,7 +174,6 @@ namespace epee
if(!strg_out.store_to_binary(buff_out))
{
LOG_ERROR("Failed to store_to_binary in command" << command);
return -1;
}
@ -202,13 +186,11 @@ namespace epee
serialization::portable_storage strg;
if(!strg.load_from_binary(in_buff))
{
LOG_ERROR("Failed to load_from_binary in notify " << command);
return -1;
}
t_in_type in_struct{};
if (!in_struct.load(strg))
{
LOG_ERROR("Failed to load in_struct in notify " << command);
return -1;
}
return cb(command, in_struct, context);
@ -294,7 +276,6 @@ namespace epee
#define END_INVOKE_MAP2() \
LOG_ERROR("Unknown command:" << command); \
return LEVIN_ERROR_CONNECTION_HANDLER_NOT_DEFINED; \
}
}

1
contrib/epee/include/epee/storages/portable_storage_base.h
View File

@ -33,6 +33,7 @@
#include <vector>
#include <deque>
#include <cstdint>
#include <map>
#include "../misc_log_ex.h"
#include "../int-util.h"

6
contrib/epee/include/epee/storages/portable_storage_from_json.h
View File

@ -398,14 +398,8 @@ namespace epee
run_handler(nullptr, it, buff_json.end(), stg, 0);
return true;
}
catch(const std::exception& ex)
{
MERROR("Failed to parse json, what: " << ex.what());
return false;
}
catch(...)
{
MERROR("Failed to parse json");
return false;
}
}

3
contrib/epee/include/epee/storages/portable_storage_to_bin.h
View File

@ -31,7 +31,8 @@
#include "../pragma_comp_defs.h"
#include "portable_storage_base.h"
#include <oxenc/endian.h>
#include <oxenmq/variant.h>
#include <oxenc/variant.h>
#include <limits>
namespace epee
{

1
contrib/epee/include/epee/storages/portable_storage_val_converters.h
View File

@ -92,7 +92,6 @@ POP_WARNINGS
void operator()(const std::string& from, uint64_t& to)
{
MTRACE("Converting std::string to uint64_t. Source: " << from);
const auto* strend = from.data() + from.size();
if (auto [p, ec] = std::from_chars(from.data(), strend, to); ec == std::errc{} && p == strend)
return; // Good: successfully consumed the whole string.

12
contrib/epee/include/epee/string_tools.h
View File

@ -215,18 +215,6 @@ POP_WARNINGS
trim(str);
return str;
}
//----------------------------------------------------------------------------
inline std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false)
{
if (s.size() < n)
{
if (prepend)
s = std::string(n - s.size(), c) + s;
else
s.append(n - s.size(), c);
}
return s;
}
}
}
#endif //_STRING_TOOLS_H_

2
contrib/epee/src/CMakeLists.txt
View File

@ -32,7 +32,6 @@ add_library(epee
levin_base.cpp
memwipe.c
mlocker.cpp
mlog.cpp
net_utils_base.cpp
network_throttle.cpp
network_throttle-detail.cpp
@ -63,7 +62,6 @@ endif()
target_link_libraries(epee
PUBLIC
easylogging
oxenmq::oxenmq
oxenc::oxenc
PRIVATE

15
contrib/epee/src/connection_basic.cpp
View File

@ -137,8 +137,6 @@ connection_basic::connection_basic(boost::asio::ip::tcp::socket&& sock, std::sha
std::string remote_addr_str = "?";
try { boost::system::error_code e; remote_addr_str = socket_.remote_endpoint(e).address().to_string(); } catch(...){} ;
MDEBUG("Spawned connection #"<<mI->m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_state->sock_count);
}
connection_basic::connection_basic(boost::asio::io_service &io_service, std::shared_ptr<connection_basic_shared_state> state)
@ -159,8 +157,6 @@ connection_basic::connection_basic(boost::asio::io_service &io_service, std::sha
std::string remote_addr_str = "?";
try { boost::system::error_code e; remote_addr_str = socket().remote_endpoint(e).address().to_string(); } catch(...){} ;
MDEBUG("Spawned connection #"<<mI->m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_state->sock_count);
}
connection_basic::~connection_basic() noexcept(false) {
@ -168,7 +164,6 @@ connection_basic::~connection_basic() noexcept(false) {
std::string remote_addr_str = "?";
try { boost::system::error_code e; remote_addr_str = socket().remote_endpoint(e).address().to_string(); } catch(...){} ;
MDEBUG("Destructing connection #"<<mI->m_peer_number << " to " << remote_addr_str);
}
void connection_basic::set_rate_up_limit(uint64_t limit) {
@ -221,7 +216,6 @@ void connection_basic::sleep_before_packet(size_t packet_size, int phase, int q
do
{ // rate limiting
if (m_was_shutdown) {
MDEBUG("m_was_shutdown - so abort sleep");
return;
}
@ -232,8 +226,7 @@ void connection_basic::sleep_before_packet(size_t packet_size, int phase, int q
delay *= 0.50;
if (delay > 0) {
long int ms = (long int)(delay * 1000);
MTRACE("Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<packet_size); // debug sleep
long int ms = (long int)(delay * 1000);
std::this_thread::sleep_for(std::chrono::milliseconds{ms});
}
} while(delay > 0);
@ -247,13 +240,11 @@ void connection_basic::sleep_before_packet(size_t packet_size, int phase, int q
}
void connection_basic::do_send_handler_write(const void* ptr , size_t cb ) {
// No sleeping here; sleeping is done once and for all in connection<t_protocol_handler>::handle_write
MTRACE("handler_write (direct) - before ASIO write, for packet="<<cb<<" B (after sleep)");
// No sleeping here; sleeping is done once and for all in connection<t_protocol_handler>::handle_write
}
void connection_basic::do_send_handler_write_from_queue( const boost::system::error_code& e, size_t cb, int q_len ) {
// No sleeping here; sleeping is done once and for all in connection<t_protocol_handler>::handle_write
MTRACE("handler_write (after write, from queue="<<q_len<<") - before ASIO write, for packet="<<cb<<" B (after sleep)");
// No sleeping here; sleeping is done once and for all in connection<t_protocol_handler>::handle_write
}
void connection_basic::logger_handle_net_read(size_t size) { // network data read

17
contrib/epee/src/mlocker.cpp
View File

@ -45,9 +45,6 @@
#include <mutex>
#include <utility>
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "mlocker"
// did an mlock operation previously fail? we only
// want to log an error once and be done with it
static std::atomic<bool> previously_failed{ false };
@ -58,7 +55,6 @@ static size_t query_page_size()
long ret = sysconf(_SC_PAGESIZE);
if (ret <= 0)
{
MERROR("Failed to determine page size");
return 0;
}
return ret;
@ -72,8 +68,6 @@ static void do_lock(void *ptr, size_t len)
{
#if defined HAVE_MLOCK
int ret = mlock(ptr, len);
if (ret < 0 && !previously_failed.exchange(true))
MERROR("Error locking page at " << ptr << ": " << strerror(errno) << ", subsequent mlock errors will be silenced");
#else
#warning Missing do_lock implementation
#endif
@ -83,11 +77,6 @@ static void do_unlock(void *ptr, size_t len)
{
#if defined HAVE_MLOCK
int ret = munlock(ptr, len);
// check whether we previously failed, but don't set it, this is just
// to pacify the errors of mlock()ing failed, in which case unlocking
// is also not going to work of course
if (ret < 0 && !previously_failed.load())
MERROR("Error unlocking page at " << ptr << ": " << strerror(errno));
#else
#warning Missing implementation of page size detection
#endif
@ -214,11 +203,7 @@ namespace epee
{
#if defined(HAVE_MLOCK)
std::map<size_t, unsigned int>::iterator i = map().find(page);
if (i == map().end())
{
MERROR("Attempt to unlock unlocked page at " << (void*)(page * page_size));
}
else
if (i != map().end())
{
if (!--i->second)
{

494
contrib/epee/src/mlog.cpp
View File

@ -1,494 +0,0 @@
// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of the Andrey N. Sabelnikov nor the
// names of its contributors may be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY
// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
#ifndef _MLOG_H_
#define _MLOG_H_
#include <date/date.h>
#include <chrono>
#ifdef _WIN32
#include <windows.h>
#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
#endif
#endif
#include <ctime>
#include <atomic>
#include <boost/algorithm/string.hpp>
#include "epee/string_tools.h"
#include "epee/misc_log_ex.h"
#ifndef USE_GHC_FILESYSTEM
#include <filesystem>
namespace fs { using namespace std::filesystem; }
#else
#include <ghc/filesystem.hpp>
namespace fs = ghc::filesystem;
#endif
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "logging"
#define MLOG_BASE_FORMAT "%datetime{%Y-%M-%d %H:%m:%s.%g}\t%thread\t%level\t%logger\t%loc\t%msg"
#define MLOG_LOG(x) CINFO(el::base::Writer,el::base::DispatchAction::FileOnlyLog,OXEN_DEFAULT_LOG_CATEGORY) << x
using namespace epee;
std::string mlog_get_default_log_path(const char *default_filename)
{
std::string process_name = epee::string_tools::get_current_module_name();
std::string default_log_folder = epee::string_tools::get_current_module_folder();
std::string default_log_file = process_name;
std::string::size_type a = default_log_file.rfind('.');
if ( a != std::string::npos )
default_log_file.erase( a, default_log_file.size());
if ( ! default_log_file.empty() )
default_log_file += ".log";
else
default_log_file = default_filename;
return (fs::u8path(default_log_folder) / fs::u8path(default_log_file)).u8string();
}
static void mlog_set_common_prefix()
{
static const char * const expected_filename = "contrib/epee/src/mlog.cpp";
const char *path = __FILE__, *expected_ptr = strstr(path, expected_filename);
if (!expected_ptr)
return;
el::Loggers::setFilenameCommonPrefix(std::string(path, expected_ptr - path));
}
static const char *get_default_categories(int level)
{
const char *categories = "";
switch (level)
{
case 0:
categories = "*:WARNING,net:FATAL,net.http:FATAL,net.p2p:FATAL,net.cn:FATAL,global:INFO,verify:FATAL,serialization:FATAL,logging:INFO,msgwriter:INFO";
break;
case 1:
categories = "*:INFO,global:INFO,stacktrace:INFO,logging:INFO,msgwriter:INFO,perf.*:DEBUG";
break;
case 2:
categories = "*:DEBUG";
break;
case 3:
categories = "*:TRACE";
break;
case 4:
categories = "*:TRACE";
break;
default:
break;
}
return categories;
}
#ifdef WIN32
bool EnableVTMode()
{
// Set output mode to handle virtual terminal sequences
HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE);
if (hOut == INVALID_HANDLE_VALUE)
{
return false;
}
DWORD dwMode = 0;
if (!GetConsoleMode(hOut, &dwMode))
{
return false;
}
dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
if (!SetConsoleMode(hOut, dwMode))
{
return false;
}
return true;
}
#endif
void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size, const std::size_t max_log_files)
{
el::Configurations c;
c.setGlobally(el::ConfigurationType::Filename, filename_base);
c.setGlobally(el::ConfigurationType::ToFile, "true");
const char *log_format = getenv("OXEN_LOG_FORMAT");
if (!log_format)
log_format = MLOG_BASE_FORMAT;
c.setGlobally(el::ConfigurationType::Format, log_format);
c.setGlobally(el::ConfigurationType::ToStandardOutput, console ? "true" : "false");
c.setGlobally(el::ConfigurationType::MaxLogFileSize, std::to_string(max_log_file_size));
el::Loggers::setDefaultConfigurations(c, true);
el::Loggers::addFlag(el::LoggingFlag::HierarchicalLogging);
el::Loggers::addFlag(el::LoggingFlag::CreateLoggerAutomatically);
el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog);
el::Loggers::addFlag(el::LoggingFlag::ColoredTerminalOutput);
el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
el::Helpers::installPreRollOutCallback([filename_base, max_log_files](const char *name, size_t){
std::string rname = filename_base + "-" + date::format("%Y-%m-%d-%H-%M-%S", std::chrono::system_clock::now());
int ret = rename(name, rname.c_str());
if (ret < 0)
{
// can't log a failure, but don't do the file removal below
return;
}
if (max_log_files != 0)
{
std::vector<fs::path> found_files;
const auto filename_base_path = fs::u8path(filename_base);
const auto parent_path = filename_base_path.has_parent_path() ? filename_base_path.parent_path() : fs::path(".");
for (const auto& p : fs::directory_iterator{parent_path})
{
const std::string filename = p.path().u8string();
if (filename.size() >= filename_base.size() && std::memcmp(filename.data(), filename_base.data(), filename_base.size()) == 0)
found_files.push_back(p.path());
}
if (found_files.size() >= max_log_files)
{
std::sort(found_files.begin(), found_files.end(), [](auto& a, auto& b) {
std::error_code ec;
return fs::last_write_time(a, ec) < fs::last_write_time(b, ec);
});
for (size_t i = 0; i <= found_files.size() - max_log_files; ++i)
{
std::error_code ec;
if (!fs::remove(found_files[i], ec))
MERROR("Failed to remove " << found_files[i] << ": " << ec.message());
}
}
}
});
mlog_set_common_prefix();
const char *loki_log = getenv("OXEN_LOGS");
if (!loki_log)
{
loki_log = get_default_categories(0);
}
mlog_set_log(loki_log);
#ifdef WIN32
EnableVTMode();
#endif
}
void mlog_set_categories(const char *categories)
{
std::string new_categories;
if (*categories)
{
if (*categories == '+')
{
++categories;
new_categories = mlog_get_categories();
if (*categories)
{
if (!new_categories.empty())
new_categories += ",";
new_categories += categories;
}
}
else if (*categories == '-')
{
++categories;
new_categories = mlog_get_categories();
std::vector<std::string> single_categories;
boost::split(single_categories, categories, boost::is_any_of(","), boost::token_compress_on);
for (const std::string &s: single_categories)
{
size_t pos = new_categories.find(s);
if (pos != std::string::npos)
new_categories = new_categories.erase(pos, s.size());
}
}
else
{
new_categories = categories;
}
}
el::Loggers::setCategories(new_categories.c_str(), true);
MLOG_LOG("New log categories: " << el::Loggers::getCategories());
}
std::string mlog_get_categories()
{
return el::Loggers::getCategories();
}
// maps epee style log level to new logging system
void mlog_set_log_level(int level)
{
const char *categories = get_default_categories(level);
mlog_set_categories(categories);
}
void mlog_set_log(const char *log)
{
long level;
char *ptr = NULL;
if (!*log)
{
mlog_set_categories(log);
return;
}
level = strtol(log, &ptr, 10);
if (ptr && *ptr)
{
// we can have a default level, eg, 2,foo:ERROR
if (*ptr == ',') {
std::string new_categories = std::string(get_default_categories(level)) + ptr;
mlog_set_categories(new_categories.c_str());
}
else {
mlog_set_categories(log);
}
}
else if (level >= 0 && level <= 4)
{
mlog_set_log_level(level);
}
else
{
MERROR("Invalid numerical log level: " << log);
}
}
namespace epee
{
bool is_stdout_a_tty()
{
static std::atomic<bool> initialized(false);
static std::atomic<bool> is_a_tty(false);
if (!initialized.load(std::memory_order_acquire))
{
#if defined(WIN32)
is_a_tty.store(0 != _isatty(_fileno(stdout)), std::memory_order_relaxed);
#else
is_a_tty.store(0 != isatty(fileno(stdout)), std::memory_order_relaxed);
#endif
initialized.store(true, std::memory_order_release);
}
return is_a_tty.load(std::memory_order_relaxed);
}
void set_console_color(int color, bool bright)
{
if (!is_stdout_a_tty())
return;
switch(color)
{
case console_color_default:
{
#ifdef WIN32
HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE| (bright ? FOREGROUND_INTENSITY:0));
#else
if(bright)
std::cout << "\033[1;37m";
else
std::cout << "\033[0m";
#endif
}
break;
case console_color_white:
{
#ifdef WIN32
HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE | (bright ? FOREGROUND_INTENSITY:0));
#else
if(bright)
std::cout << "\033[1;37m";
else
std::cout << "\033[0;37m";
#endif
}
break;
case console_color_red:
{
#ifdef WIN32
HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | (bright ? FOREGROUND_INTENSITY:0));
#else
if(bright)
std::cout << "\033[1;31m";
else
std::cout << "\033[0;31m";
#endif
}
break;
case console_color_green:
{
#ifdef WIN32
HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | (bright ? FOREGROUND_INTENSITY:0));
#else
if(bright)
std::cout << "\033[1;32m";
else
std::cout << "\033[0;32m";
#endif
}
break;
case console_color_blue:
{
#ifdef WIN32
HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_INTENSITY);//(bright ? FOREGROUND_INTENSITY:0));
#else
if(bright)
std::cout << "\033[1;34m";
else
std::cout << "\033[0;34m";
#endif
}
break;
case console_color_cyan:
{
#ifdef WIN32
HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | (bright ? FOREGROUND_INTENSITY:0));
#else
if(bright)
std::cout << "\033[1;36m";
else
std::cout << "\033[0;36m";
#endif
}
break;
case console_color_magenta:
{
#ifdef WIN32
HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_RED | (bright ? FOREGROUND_INTENSITY:0));
#else
if(bright)
std::cout << "\033[1;35m";
else
std::cout << "\033[0;35m";
#endif
}
break;
case console_color_yellow:
{
#ifdef WIN32
HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | (bright ? FOREGROUND_INTENSITY:0));
#else
if(bright)
std::cout << "\033[1;33m";
else
std::cout << "\033[0;33m";
#endif
}
break;
}
}
void reset_console_color() {
if (!is_stdout_a_tty())
return;
#ifdef WIN32
HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE);
#else
std::cout << "\033[0m";
std::cout.flush();
#endif
}
}
static bool mlog(el::Level level, const char *category, const char *format, va_list ap) noexcept
{
int size = 0;
char *p = NULL;
va_list apc;
bool ret = true;
/* Determine required size */
va_copy(apc, ap);
size = vsnprintf(p, size, format, apc);
va_end(apc);
if (size < 0)
return false;
size++; /* For '\0' */
p = (char*)malloc(size);
if (p == NULL)
return false;
size = vsnprintf(p, size, format, ap);
if (size < 0)
{
free(p);
return false;
}
try
{
/* TODO(loki): when pulling upstream epee changes change this to:
MCLOG(level, category, el::Color::Default, p);
*/
MCLOG(level, category, p);
}
catch(...)
{
ret = false;
}
free(p);
return ret;
}
#define DEFLOG(fun,lev) \
bool m##fun(const char *category, const char *fmt, ...) { va_list ap; va_start(ap, fmt); bool ret = mlog(el::Level::lev, category, fmt, ap); va_end(ap); return ret; }
DEFLOG(error, Error)
DEFLOG(warning, Warning)
DEFLOG(info, Info)
DEFLOG(debug, Debug)
DEFLOG(trace, Trace)
#undef DEFLOG
#endif //_MLOG_H_

28
contrib/epee/src/network_throttle-detail.cpp
View File

@ -133,8 +133,7 @@ void network_throttle::set_name(const std::string &name)
void network_throttle::set_target_speed( network_speed_kbps target )
{
m_target_speed = target * 1024;
MINFO("Setting LIMIT: " << target << " kbps");
m_target_speed = target * 1024;
}
network_speed_kbps network_throttle::get_target_speed()
@ -155,7 +154,6 @@ void network_throttle::tick()
// TODO optimize when moving few slots at once
while ( (!m_any_packet_yet) || (last_sample_time_slot < current_sample_time_slot))
{
MTRACE("Moving counter buffer by 1 second " << last_sample_time_slot << " < " << current_sample_time_slot << " (last time " << m_last_sample_time<<")");
// rotate buffer
m_history.push_front(packet_info());
if (! m_any_packet_yet)
@ -185,13 +183,6 @@ void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_s
std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends;
std::string history_str = oss.str();
MTRACE("Throttle " << m_name << ": packet of ~"<<packet_size<<"b " << " (from "<<orginal_size<<" b)"
<< " Speed AVG=" << std::setw(4) << ((long int)(cts .average/1024)) <<"[w="<<cts .window<<"]"
<< " " << std::setw(4) << ((long int)(cts2.average/1024)) <<"[w="<<cts2.window<<"]"
<<" / " << " Limit="<< ((long int)(m_target_speed/1024)) <<" KiB/sec "
<< " " << history_str
);
}
void network_throttle::handle_trafic_tcp(size_t packet_size)
@ -214,8 +205,6 @@ void network_throttle::logger_handle_net(const std::string &filename, double tim
std::fstream file;
file.open(filename.c_str(), std::ios::app | std::ios::out );
file.precision(6);
if(!file.is_open())
MWARNING("Can't open file " << filename);
file << static_cast<int>(time) << " " << static_cast<double>(size/1024) << "\n";
file.close();
}
@ -269,28 +258,13 @@ void network_throttle::calculate_times(size_t packet_size, calculate_times_struc
if (cts.delay>=0) cts.delay = 0; // no traffic in history so we will not wait
}
double Wgood=-1;
{ // how much data we recommend now to download
Wgood = the_window_size + 1;
cts.recomendetDataSize = M*cts.window - E;
}
if (dbg) {
std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends;
std::string history_str = oss.str();
MTRACE((cts.delay > 0 ? "SLEEP" : "")
<< "dbg " << m_name << ": "
<< "speed is A=" << std::setw(8) <<cts.average<<" vs "
<< "Max=" << std::setw(8) <<M<<" "
<< " so sleep: "
<< "D=" << std::setw(8) <<cts.delay<<" sec "
<< "E="<< std::setw(8) << E << " (Enow="<<std::setw(8)<<Enow<<") "
<< "M=" << std::setw(8) << M <<" W="<< std::setw(8) << cts.window << " "
<< "R=" << std::setw(8) << cts.recomendetDataSize << " Wgood" << std::setw(8) << Wgood << " "
<< "History: " << std::setw(8) << history_str << " "
<< "m_last_sample_time=" << std::setw(8) << m_last_sample_time
);
}
}

4
contrib/epee/src/parserse_base_utils.cpp
View File

@ -1,7 +1,4 @@
#include "epee/storages/parserse_base_utils.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "serialization"
#include <algorithm>
#include <string>
@ -99,7 +96,6 @@ void match_string2(const char*& star_end_string, const char* buf_end, std::strin
break;
default:
val.push_back(*it);
LOG_PRINT_L0("Unknown escape sequence :\"\\" << *it << "\"");
}
escape_mode = false;
}else if(*it == '"')

5
contrib/epee/src/portable_storage.cpp
View File

@ -1,6 +1,6 @@
#include "epee/storages/portable_storage_to_json.h"
#include "epee/storages/portable_storage.h"
#include <oxenmq/variant.h>
#include <oxenc/variant.h>
namespace epee {
namespace serialization {
@ -119,7 +119,6 @@ namespace epee {
m_root.m_entries.clear();
if(source.size() < sizeof(storage_block_header))
{
LOG_ERROR("portable_storage: wrong binary format, packet size = " << source.size() << " less than expected sizeof(storage_block_header)=" << sizeof(storage_block_header));
return false;
}
storage_block_header* pbuff = (storage_block_header*)source.data();
@ -127,12 +126,10 @@ namespace epee {
pbuff->m_signature_b != PORTABLE_STORAGE_SIGNATUREB
)
{
LOG_ERROR("portable_storage: wrong binary format - signature mismatch");
return false;
}
if(pbuff->m_ver != PORTABLE_STORAGE_FORMAT_VER)
{
LOG_ERROR("portable_storage: wrong binary format - unknown format ver = " << pbuff->m_ver);
return false;
}
TRY_ENTRY();

1
contrib/epee/src/wipeable_string.cpp
View File

@ -37,6 +37,7 @@
#include "epee/memwipe.h"
#include "epee/misc_log_ex.h"
#include "epee/wipeable_string.h"
#include <limits>
static constexpr const char hex[] = u8"0123456789abcdef";

2
contrib/format.sh
View File

@ -2,7 +2,7 @@
CLANG_FORMAT_DESIRED_VERSION=11
TARGET_DIRS=(src/wallet3 src/sqlitedb)
TARGET_DIRS=(src/wallet3 src/sqlitedb src/logging)
binary=$(which clang-format-$CLANG_FORMAT_DESIRED_VERSION 2>/dev/null)
if [ $? -ne 0 ]; then

8
external/CMakeLists.txt vendored
View File

@ -60,17 +60,19 @@ macro(system_or_submodule BIGNAME smallname pkgconf subdir)
endmacro()
system_or_submodule(OXENC oxenc liboxenc>=1.0.3 oxen-encoding)
system_or_submodule(OXENMQ oxenmq liboxenmq>=1.2.12 oxen-mq)
system_or_submodule(FMT fmt fmt>=8.0.0 fmt)
system_or_submodule(OXENMQ oxenmq liboxenmq>=1.2.13 oxen-mq)
add_subdirectory(db_drivers)
add_subdirectory(easylogging++ easyloggingpp)
add_subdirectory(randomx EXCLUDE_FROM_ALL)
add_subdirectory(date EXCLUDE_FROM_ALL)
set(JSON_BuildTests OFF CACHE INTERNAL "")
set(JSON_MultipleHeaders ON CACHE BOOL "") # Allows multi-header nlohmann use
add_subdirectory(nlohmann-json EXCLUDE_FROM_ALL)
if(BUILD_STATIC_DEPS)
set(OXEN_LOGGING_FORCE_SUBMODULES TRUE CACHE BOOL "" FORCE)
endif()
add_subdirectory(oxen-logging)
# uSockets doesn't really have a proper build system (just a very simple Makefile) so build it
# ourselves.

61
external/easylogging++/CMakeLists.txt vendored
View File

@ -1,61 +0,0 @@
# Copyright (c) 2014-2018, The Monero Project
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification, are
# permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of
# conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list
# of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cmake_minimum_required(VERSION 3.5)
project(easylogging CXX)
add_library(easylogging
easylogging++.cc)
set_property(TARGET easylogging PROPERTY POSITION_INDEPENDENT_CODE ON)
set_property(TARGET easylogging PROPERTY CXX_STANDARD 11)
set_property(TARGET easylogging PROPERTY CXX_STANDARD_REQUIRED ON)
find_package(Threads)
find_package(Backtrace)
target_include_directories(easylogging PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}")
target_link_libraries(easylogging
PRIVATE
Threads::Threads
${Backtrace_LIBRARIES})
# GUI/libwallet install target
if (BUILD_GUI_DEPS)
if(IOS)
set(lib_folder lib-${ARCH})
target_compile_options(easylogging PUBLIC -fno-stack-check)
else()
set(lib_folder lib)
endif()
install(TARGETS easylogging
ARCHIVE DESTINATION ${lib_folder}
LIBRARY DESTINATION ${lib_folder})
endif()

17
external/easylogging++/ea_config.h vendored
View File

@ -1,17 +0,0 @@
#pragma once
#include <limits.h>
#define ELPP_THREAD_SAFE
#define ELPP_DEFAULT_LOG_FILE ""
#define ELPP_DISABLE_DEFAULT_CRASH_HANDLING
#define ELPP_NO_CHECK_MACROS
#define ELPP_WINSOCK2
#define ELPP_NO_DEBUG_MACROS
#define ELPP_UTC_DATETIME
#ifdef EASYLOGGING_CC
#if !(!defined __GLIBC__ || !defined __GNUC__ || defined __MINGW32__ || defined __MINGW64__ || defined __ANDROID__)
#define ELPP_FEATURE_CRASH_LOG
#endif
#endif

3319
external/easylogging++/easylogging++.cc vendored
View File

File diff suppressed because it is too large Load Diff

4653
external/easylogging++/easylogging++.h vendored
View File

File diff suppressed because it is too large Load Diff

1
external/fmt vendored

@ -1 +0,0 @@
Subproject commit d141cdbeb0fb422a3fb7173b285fd38e0d1772dc

1
external/oxen-logging vendored Submodule

@ -0,0 +1 @@
Subproject commit 1c00cfa9aef0b8e449a1746e6d2af87881910c45

2
external/oxen-mq vendored

@ -1 +1 @@
Subproject commit a93c16af04eba58fba2bf7f5726c4669c290bd46
Subproject commit 0858dd278b91a899b69210088622ca6fa3bb0eed

1
src/CMakeLists.txt
View File

@ -53,6 +53,7 @@ add_subdirectory(ringct)
add_subdirectory(checkpoints)
add_subdirectory(cryptonote_basic)
add_subdirectory(cryptonote_core)
add_subdirectory(logging)
add_subdirectory(lmdb)
add_subdirectory(multisig)
add_subdirectory(net)

1
src/blockchain_db/CMakeLists.txt
View File

@ -43,6 +43,7 @@ target_link_libraries(blockchain_db
lmdb
filesystem
Boost::thread
logging
extra)
target_compile_definitions(blockchain_db PRIVATE

34
src/blockchain_db/blockchain_db.cpp
View File

@ -40,12 +40,11 @@
#include "lmdb/db_lmdb.h"
#include <chrono>
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "blockchain.db"
namespace cryptonote
{
static auto logcat = log::Cat("blockchain.db");
const command_line::arg_descriptor<std::string> arg_db_sync_mode = {
"db-sync-mode"
, "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[<nblocks_per_sync>[blocks]|<nbytes_per_sync>[bytes]]."
@ -85,7 +84,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair
{
// should only need to compute hash for miner transactions
tx_hash = get_transaction_hash(tx);
LOG_PRINT_L3("null tx_hash_ptr - needed to compute: " << tx_hash);
log::trace(logcat, "null tx_hash_ptr - needed to compute: {}", tx_hash);
}
else
{
@ -119,7 +118,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair
}
else
{
LOG_PRINT_L1("Unsupported input type, removing key images and aborting transaction addition");
log::info(logcat, "Unsupported input type, removing key images and aborting transaction addition");
for (const txin_v& tx_input : tx.vin)
{
if (std::holds_alternative<txin_to_key>(tx_input))
@ -202,7 +201,7 @@ uint64_t BlockchainDB::add_block( const std::pair<block, std::string>& blck
num_rct_outs += blk.miner_tx.vout.size();
int tx_i = 0;
crypto::hash tx_hash = crypto::null_hash;
crypto::hash tx_hash{};
for (const std::pair<transaction, std::string>& tx : txs)
{
tx_hash = blk.tx_hashes[tx_i];
@ -330,22 +329,21 @@ void BlockchainDB::reset_stats()
void BlockchainDB::show_stats()
{
LOG_PRINT_L1("\n"
<< "*********************************\n"
<< "num_calls: " << num_calls << "\n"
<< "time_blk_hash: " << tools::friendly_duration(time_blk_hash) << "\n"
<< "time_tx_exists: " << tools::friendly_duration(time_tx_exists) << "\n"
<< "time_add_block1: " << tools::friendly_duration(time_add_block1) << "\n"
<< "time_add_transaction: " << tools::friendly_duration(time_add_transaction) << "\n"
<< "time_commit1: " << tools::friendly_duration(time_commit1) << "\n"
<< "*********************************\n"
);
log::info(logcat, "\n*********************************\n \
num_calls: {}\n \
time_blk_hash: {}\n \
time_tx_exists: {}\n \
time_add_block1: {}\n \
time_add_transaction: {}\n \
time_commit1: {}\n \
*********************************\n",
num_calls, tools::friendly_duration(time_blk_hash), tools::friendly_duration(time_tx_exists), tools::friendly_duration(time_add_block1), tools::friendly_duration(time_add_transaction), tools::friendly_duration(time_commit1));
}
void BlockchainDB::fixup(cryptonote::network_type)
{
if (is_read_only()) {
LOG_PRINT_L1("Database is opened read only - skipping fixup check");
log::info(logcat, "Database is opened read only - skipping fixup check");
return;
}
@ -393,7 +391,7 @@ uint64_t BlockchainDB::get_tx_block_height(const crypto::hash &h) const
if (result == std::numeric_limits<uint64_t>::max())
{
std::string err = "tx_data_t with hash " + tools::type_to_hex(h) + " not found in db";
LOG_PRINT_L1(err);
log::info(logcat, "{}", err);
throw TX_DNE(std::move(err));
}
return result;

543
src/blockchain_db/lmdb/db_lmdb.cpp
View File

File diff suppressed because it is too large Load Diff

6
src/blockchain_db/locked_txn.h
View File

@ -30,6 +30,8 @@
#pragma once
#include "cryptonote_core/blockchain.h"
namespace cryptonote
{
// This class is meant to create a batch when none currently exists.
@ -46,8 +48,8 @@ namespace cryptonote
LockedTXN(LockedTXN &&o) : m_db{o.m_db}, m_batch{o.m_batch} { o.m_batch = false; }
LockedTXN &operator=(LockedTXN &&) = delete;
void commit() { try { if (m_batch) { m_db.batch_stop(); m_batch = false; } } catch (const std::exception &e) { MWARNING("LockedTXN::commit filtering exception: " << e.what()); } }
void abort() { try { if (m_batch) { m_db.batch_abort(); m_batch = false; } } catch (const std::exception &e) { MWARNING("LockedTXN::abort filtering exception: " << e.what()); } }
void commit() { try { if (m_batch) { m_db.batch_stop(); m_batch = false; } } catch (const std::exception &e) { log::warning(globallogcat, "LockedTXN::commit filtering exception: {}", e.what()); } }
void abort() { try { if (m_batch) { m_db.batch_abort(); m_batch = false; } } catch (const std::exception &e) { log::warning(globallogcat, "LockedTXN::abort filtering exception: {}", e.what()); } }
~LockedTXN() { this->abort(); }
private:
BlockchainDB &m_db;

85
src/blockchain_db/sqlite/db_sqlite.cpp
View File

@ -31,7 +31,6 @@
#include <sodium.h>
#include <fmt/core.h>
#include <iostream>
#include <cassert>
#include "cryptonote_config.h"
@ -40,13 +39,12 @@
#include "common/string_util.h"
#include "cryptonote_basic/hardfork.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "blockchain.db.sqlite"
namespace cryptonote {
static auto logcat = log::Cat("blockchain.db.sqlite");
BlockchainSQLite::BlockchainSQLite(cryptonote::network_type nettype, fs::path db_path): db::Database(db_path, ""), m_nettype(nettype), filename {db_path.u8string()} {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__);
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
height = 0;
if (!db.tableExists("batched_payments_accrued") || !db.tableExists("batched_payments_raw") || !db.tableExists("batch_db_info")) {
@ -59,7 +57,7 @@ namespace cryptonote {
}
void BlockchainSQLite::create_schema() {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__);
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
auto& netconf = cryptonote::get_config(m_nettype);
@ -118,7 +116,7 @@ namespace cryptonote {
)",
netconf.BATCHING_INTERVAL));
MDEBUG("Database setup complete");
log::debug(logcat, "Database setup complete");
}
void BlockchainSQLite::upgrade_schema() {
@ -131,7 +129,7 @@ namespace cryptonote {
}
if (!have_offset) {
MINFO("Adding payout_offset to batching db");
log::info(logcat, "Adding payout_offset to batching db");
auto& netconf = get_config(m_nettype);
SQLite::Transaction transaction{
db,
@ -166,7 +164,7 @@ namespace cryptonote {
if (count != 0) {
constexpr auto error = "Batching db update to add offsets failed: not all addresses were converted";
MFATAL(error);
log::error(logcat, error);
throw std::runtime_error{error};
}
@ -175,7 +173,7 @@ namespace cryptonote {
}
void BlockchainSQLite::reset_database() {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__);
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
db.exec(R"(
DROP TABLE IF EXISTS batched_payments_accrued;
@ -189,11 +187,11 @@ namespace cryptonote {
create_schema();
MDEBUG("Database reset complete");
log::debug(logcat, "Database reset complete");
}
void BlockchainSQLite::update_height(uint64_t new_height) {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__ << " Called with new height: " << new_height);
log::trace(logcat, "BlockchainDB_SQLITE::{} Called with new height: {}", __func__, new_height);
height = new_height;
prepared_exec(
"UPDATE batch_db_info SET height = ?",
@ -201,12 +199,12 @@ namespace cryptonote {
}
void BlockchainSQLite::increment_height() {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__ << " Called with height: " << height + 1);
log::trace(logcat, "BlockchainDB_SQLITE::{} Called with height: {}", __func__, height + 1);
update_height(height + 1);
}
void BlockchainSQLite::decrement_height() {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__ << " Called with height: " << height - 1);
log::trace(logcat, "BlockchainDB_SQLITE::{} Called with height: {}", __func__, height - 1);
update_height(height - 1);
}
@ -221,7 +219,7 @@ namespace cryptonote {
bool BlockchainSQLite::add_sn_rewards(const std::vector<cryptonote::batch_sn_payment>& payments) {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__);
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
auto insert_payment = prepared_st(
"INSERT INTO batched_payments_accrued (address, payout_offset, amount) VALUES (?, ?, ?)"
" ON CONFLICT (address) DO UPDATE SET amount = amount + excluded.amount");
@ -232,8 +230,7 @@ namespace cryptonote {
auto offset = static_cast<int>(payment.address_info.address.modulus(netconf.BATCHING_INTERVAL));
auto amt = static_cast<int64_t>(payment.amount);
const auto& address_str = get_address_str(payment.address_info.address);
MTRACE(fmt::format("Adding record for SN reward contributor {} to database with amount {}",
address_str, amt));
log::trace(logcat, "Adding record for SN reward contributor {} to database with amount {}", address_str, amt);
db::exec_query(insert_payment, address_str, offset, amt);
insert_payment->reset();
}
@ -242,7 +239,7 @@ namespace cryptonote {
}
bool BlockchainSQLite::subtract_sn_rewards(const std::vector<cryptonote::batch_sn_payment>& payments) {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__);
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
auto update_payment = prepared_st(
"UPDATE batched_payments_accrued SET amount = (amount - ?) WHERE address = ?");
@ -250,7 +247,7 @@ namespace cryptonote {
const auto& address_str = get_address_str(payment.address_info.address);
auto result = db::exec_query(update_payment, static_cast<int64_t>(payment.amount), address_str);
if (!result) {
MERROR("tried to subtract payment from an address that doesn't exist: " << address_str);
log::error(logcat, "tried to subtract payment from an address that doesn't exist: {}", address_str);
return false;
}
update_payment->reset();
@ -260,7 +257,7 @@ namespace cryptonote {
}
std::vector<cryptonote::batch_sn_payment> BlockchainSQLite::get_sn_payments(uint64_t block_height) {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__);
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
// <= here because we might have crap in the db that we don't clear until we actually add the HF
// block later on. (This is a pretty slim edge case that happened on devnet and is probably
@ -289,7 +286,7 @@ namespace cryptonote {
uint64_t BlockchainSQLite::get_accrued_earnings(const std::string& address) {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__);
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
auto earnings = prepared_maybe_get<int64_t>(
"SELECT amount FROM batched_payments_accrued WHERE address = ?",
@ -298,7 +295,7 @@ namespace cryptonote {
}
std::pair<std::vector<std::string>, std::vector<uint64_t>> BlockchainSQLite::get_all_accrued_earnings() {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__);
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
std::pair<std::vector<std::string>, std::vector<uint64_t>> result;
auto& [addresses, amounts] = result;
@ -316,7 +313,7 @@ namespace cryptonote {
}
void BlockchainSQLite::calculate_rewards(hf hf_version, uint64_t distribution_amount, const service_nodes::service_node_info& sn_info, std::vector<cryptonote::batch_sn_payment>& payments) {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__);
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
// Find out how much is due for the operator: fee_portions/PORTIONS * reward
assert(sn_info.portions_for_operator <= old::STAKING_PORTIONS);
@ -376,7 +373,7 @@ namespace cryptonote {
if (uint64_t tx_fees = block_reward - service_node_reward;
tx_fees > 0
&& block.service_node_winner_key // "service_node_winner_key" tracks the pulse winner; 0 if a mined block
&& crypto_core_ed25519_is_valid_point(reinterpret_cast<const unsigned char *>(block.service_node_winner_key.data))
&& crypto_core_ed25519_is_valid_point(block.service_node_winner_key.data())
) {
if (auto service_node_winner = service_nodes_state.service_nodes_infos.find(block.service_node_winner_key);
@ -423,7 +420,7 @@ namespace cryptonote {
bool BlockchainSQLite::add_block(const cryptonote::block& block,
const service_nodes::service_node_list::state_t& service_nodes_state) {
auto block_height = get_block_height(block);
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__ << " called on height: " << block_height);
log::trace(logcat, "BlockchainDB_SQLITE::{} called on height: {}", __func__, block_height);
auto hf_version = block.major_version;
if (hf_version < hf::hf19_reward_batching) {
@ -433,13 +430,13 @@ namespace cryptonote {
auto fork_height = cryptonote::get_hard_fork_heights(m_nettype, hf::hf19_reward_batching);
if (block_height == fork_height.first.value_or(0)) {
MDEBUG("Batching of Service Node Rewards Begins");
log::debug(logcat, "Batching of Service Node Rewards Begins");
reset_database();
update_height(block_height - 1);
}
if (block_height != height + 1) {
MERROR(fmt::format("Block height ({}) out of sync with batching database ({})", block_height, height));
log::error(logcat, "Block height ({}) out of sync with batching database ({})", block_height, height);
return false;
}
@ -472,7 +469,7 @@ namespace cryptonote {
transaction.commit();
} catch (std::exception& e) {
MFATAL("Error adding reward payments: " << e.what());
log::error(logcat, "Error adding reward payments: {}", e.what());
return false;
}
return true;
@ -482,13 +479,13 @@ namespace cryptonote {
const service_nodes::service_node_list::state_t& service_nodes_state) {
auto block_height = get_block_height(block);
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__ << " called on height: " << block_height);
log::trace(logcat, "BlockchainDB_SQLITE::{} called on height: {}", __func__, block_height);
if (height < block_height) {
MDEBUG("Block above batching DB height skipping pop");
log::debug(logcat, "Block above batching DB height skipping pop");
return true;
}
if (block_height != height) {
MERROR("Block height out of sync with batching database");
log::error(logcat, "Block height out of sync with batching database");
return false;
}
@ -514,7 +511,7 @@ namespace cryptonote {
decrement_height();
transaction.commit();
} catch (std::exception& e) {
MFATAL("Error subtracting reward payments: " << e.what());
log::error(logcat, "Error subtracting reward payments: {}", e.what());
return false;
}
return true;
@ -524,10 +521,10 @@ namespace cryptonote {
const std::vector<std::tuple<crypto::public_key, uint64_t>>& miner_tx_vouts,
const std::vector<cryptonote::batch_sn_payment>& calculated_payments_from_batching_db,
uint64_t block_height) {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__);
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
if (miner_tx_vouts.size() != calculated_payments_from_batching_db.size()) {
MERROR(fmt::format("Length of batch payments ({}) does not match block vouts ({})", calculated_payments_from_batching_db.size(), miner_tx_vouts.size()));
log::error(logcat, "Length of batch payments ({}) does not match block vouts ({})", calculated_payments_from_batching_db.size(), miner_tx_vouts.size());
return false;
}
@ -546,23 +543,23 @@ namespace cryptonote {
uint64_t amount = amt * BATCH_REWARD_FACTOR;
const auto& from_db = calculated_payments_from_batching_db[vout_index];
if (amount != from_db.amount) {
MERROR(fmt::format("Batched payout amount incorrect. Should be {}, not {}", from_db.amount, amount));
log::error(logcat, "Batched payout amount incorrect. Should be {}, not {}", from_db.amount, amount);
return false;
}
crypto::public_key out_eph_public_key{};
if (!cryptonote::get_deterministic_output_key(from_db.address_info.address, deterministic_keypair, vout_index, out_eph_public_key)) {
MERROR("Failed to generate output one-time public key");
log::error(logcat, "Failed to generate output one-time public key");
return false;
}
if (tools::view_guts(pubkey) != tools::view_guts(out_eph_public_key)) {
MERROR("Output ephemeral public key does not match");
log::error(logcat, "Output ephemeral public key does not match");
return false;
}
total_oxen_payout_in_vouts += amount;
finalised_payments.emplace_back(from_db.address_info, amount);
}
if (total_oxen_payout_in_vouts != total_oxen_payout_in_our_db) {
MERROR(fmt::format("Total batched payout amount incorrect. Should be {}, not {}", total_oxen_payout_in_our_db, total_oxen_payout_in_vouts));
log::error(logcat, "Total batched payout amount incorrect. Should be {}, not {}", total_oxen_payout_in_our_db, total_oxen_payout_in_vouts);
return false;
}
@ -570,7 +567,7 @@ namespace cryptonote {
}
bool BlockchainSQLite::save_payments(uint64_t block_height, const std::vector<batch_sn_payment>& paid_amounts) {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__);
log::trace(logcat, "BlockchainDB_SQLITE::{}", __func__);
auto select_sum = prepared_st(
"SELECT amount from batched_payments_accrued WHERE address = ?");
@ -588,8 +585,7 @@ namespace cryptonote {
auto amount = static_cast<uint64_t>(*maybe_amount) / BATCH_REWARD_FACTOR * BATCH_REWARD_FACTOR;
if (amount != payment.amount) {
MERROR(fmt::format("Invalid amounts passed in to save payments for address {}: received {}, expected {} (truncated from {})",
address_str, payment.amount, amount, *maybe_amount));
log::error(logcat, "Invalid amounts passed in to save payments for address {}: received {}, expected {} (truncated from {})", address_str, payment.amount, amount, *maybe_amount);
return false;
}
@ -598,8 +594,7 @@ namespace cryptonote {
}
else {
// This shouldn't occur: we validate payout addresses much earlier in the block validation.
MERROR(fmt::format("Internal error: Invalid amounts passed in to save payments for address {}: that address has no accrued rewards",
address_str));
log::error(logcat, "Internal error: Invalid amounts passed in to save payments for address {}: that address has no accrued rewards", address_str);
return false;
}
@ -609,7 +604,7 @@ namespace cryptonote {
}
std::vector<cryptonote::batch_sn_payment> BlockchainSQLite::get_block_payments(uint64_t block_height) {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__ << " Called with height: " << block_height);
log::trace(logcat, "BlockchainDB_SQLITE::{} Called with height: {}", __func__, block_height);
std::vector<cryptonote::batch_sn_payment> payments_at_height;
auto paid = prepared_results<std::string_view, int64_t>(
@ -626,7 +621,7 @@ namespace cryptonote {
}
bool BlockchainSQLite::delete_block_payments(uint64_t block_height) {
LOG_PRINT_L3("BlockchainDB_SQLITE::" << __func__ << " Called with height: " << block_height);
log::trace(logcat, "BlockchainDB_SQLITE::{} Called with height: {}", __func__, block_height);
prepared_exec(
"DELETE FROM batched_payments_paid WHERE height_paid >= ?",
static_cast<int64_t>(block_height));

1
src/blockchain_db/sqlite/db_sqlite.h
View File

@ -30,7 +30,6 @@
#include <string>
#include <filesystem>
#include "epee/misc_log_ex.h"
#include "cryptonote_basic/cryptonote_format_utils.h"
#include "cryptonote_core/cryptonote_tx_utils.h"
#include "sqlitedb/database.hpp"

3
src/blockchain_utilities/CMakeLists.txt
View File

@ -35,6 +35,7 @@ target_link_libraries(blockchain_tools_common_libs INTERFACE
filesystem
Boost::program_options
SQLiteCpp
logging
extra)
@ -111,7 +112,7 @@ target_link_libraries(blockchain_stats PRIVATE blockchain_tools_common_libs date
if (TARGET sodium_vendor OR NOT SODIUM_VERSION VERSION_LESS 1.0.17)
oxen_add_executable(sn_key_tool "oxen-sn-keys" sn_key_tool.cpp)
target_link_libraries(sn_key_tool PRIVATE sodium oxenmq filesystem)
target_link_libraries(sn_key_tool PRIVATE sodium oxenmq filesystem fmt::fmt)
else()
message(STATUS "Not building oxen-sn-keys tool (requires libsodium >= 1.0.17)")
endif()

105
src/blockchain_utilities/blockchain_ancestry.cpp
View File

@ -30,15 +30,17 @@
#define __STDC_FORMAT_MACROS // NOTE(oxen): Explicitly define the SCNu64 macro on Mingw
#endif
#include <cinttypes>
#include <unordered_map>
#include <unordered_set>
#include <boost/archive/portable_binary_iarchive.hpp>
#include <boost/archive/portable_binary_oarchive.hpp>
#include <fstream>
#include "common/unordered_containers_boost_serialization.h"
#include "common/command_line.h"
#include "common/varint.h"
#include "common/signal_handler.h"
#include "common/fs.h"
#include "common/fs-format.h"
#include "serialization/boost_std_variant.h"
#include "cryptonote_basic/cryptonote_boost_serialization.h"
#include "cryptonote_core/cryptonote_core.h"
@ -47,12 +49,11 @@
#include "version.h"
#include "cryptonote_core/uptime_proof.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "bcutil"
namespace po = boost::program_options;
using namespace cryptonote;
static auto logcat = log::Cat("bcutil");
static bool stop_requested = false;
static uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0;
static bool opt_cache_outputs = false, opt_cache_txes = false, opt_cache_blocks = false;
@ -102,7 +103,7 @@ struct tx_data_t
vin.push_back(std::make_pair(txin->amount, cryptonote::relative_output_offsets_to_absolute(txin->key_offsets)));
else
{
LOG_PRINT_L0("Bad vin type in txid " << get_transaction_hash(tx));
log::warning(logcat, "Bad vin type in txid {}", get_transaction_hash(tx));
throw std::runtime_error("Bad vin type");
}
}
@ -116,7 +117,7 @@ struct tx_data_t
}
else
{
LOG_PRINT_L0("Bad vout type in txid " << get_transaction_hash(tx));
log::warning(logcat, "Bad vout type in txid {}", get_transaction_hash(tx));
throw std::runtime_error("Bad vout type");
}
}
@ -215,7 +216,7 @@ static std::unordered_set<ancestor> get_ancestry(const std::unordered_map<crypto
std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::const_iterator i = ancestry.find(txid);
if (i == ancestry.end())
{
//MERROR("txid ancestry not found: " << txid);
//log::error(logcat, "txid ancestry not found: {}", txid);
//throw std::runtime_error("txid ancestry not found");
return std::unordered_set<ancestor>();
}
@ -234,7 +235,7 @@ static bool get_block_from_height(ancestry_state_t &state, BlockchainDB *db, uin
std::string bd = db->get_block_blob_from_height(height);
if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
{
LOG_PRINT_L0("Bad block from db");
log::warning(logcat, "Bad block from db");
return false;
}
if (opt_cache_blocks)
@ -259,13 +260,13 @@ static bool get_transaction(ancestry_state_t &state, BlockchainDB *db, const cry
std::string bd;
if (!db->get_pruned_tx_blob(txid, bd))
{
LOG_PRINT_L0("Failed to get txid " << txid << " from db");
log::warning(logcat, "Failed to get txid {} from db", txid);
return false;
}
cryptonote::transaction tx;
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx))
{
LOG_PRINT_L0("Bad tx: " << txid);
log::warning(logcat, "Bad tx: {}", txid);
return false;
}
tx_data = ::tx_data_t(tx);
@ -304,7 +305,7 @@ static bool get_output_txid(ancestry_state_t &state, BlockchainDB *db, uint64_t
}
else
{
LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx));
log::warning(logcat, "Bad vout type in txid {}", cryptonote::get_transaction_hash(b.miner_tx));
return false;
}
}
@ -334,8 +335,6 @@ int main(int argc, char* argv[])
epee::string_tools::set_module_name_and_folder(argv[0]);
uint32_t log_level = 0;
tools::on_startup();
auto opt_size = command_line::boost_option_sizes();
@ -388,14 +387,17 @@ int main(int argc, char* argv[])
std::cout << desc_options << std::endl;
return 1;
}
mlog_configure(mlog_get_default_log_path("oxen-blockchain-ancestry.log"), true);
if (!command_line::is_arg_defaulted(vm, arg_log_level))
mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
LOG_PRINT_L0("Starting...");
auto m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir);
auto log_file_path = m_config_folder + "oxen-blockchain-ancestry.log";
log::Level log_level;
if(auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
throw std::runtime_error{"Incorrect log level"};
}
oxen::logging::init(log_file_path, log_level);
log::warning(logcat, "Starting...");
std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir);
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
@ -416,7 +418,7 @@ int main(int argc, char* argv[])
std::cerr << "Only one of --txid, --height, --output can be given" << std::endl;
return 1;
}
crypto::hash opt_txid = crypto::null_hash;
crypto::hash opt_txid{};
uint64_t output_amount = 0, output_offset = 0;
if (!opt_txid_string.empty())
{
@ -435,19 +437,19 @@ int main(int argc, char* argv[])
}
}
LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
log::warning(logcat, "Initializing source blockchain (BlockchainDB)");
blockchain_objects_t blockchain_objects = {};
Blockchain *core_storage = &blockchain_objects.m_blockchain;
BlockchainDB *db = new_db();
if (db == NULL)
{
LOG_ERROR("Failed to initialize a database");
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
LOG_PRINT_L0("database: LMDB");
log::warning(logcat, "database: LMDB");
fs::path filename = fs::u8path(opt_data_dir) / db->get_db_name();
LOG_PRINT_L0("Loading blockchain from folder " << filename << " ...");
log::warning(logcat, "Loading blockchain from folder {} ...", filename);
try
{
@ -455,20 +457,20 @@ int main(int argc, char* argv[])
}
catch (const std::exception& e)
{
LOG_PRINT_L0("Error opening database: " << e.what());
log::warning(logcat, "Error opening database: {}", e.what());
return 1;
}
r = core_storage->init(db, nullptr /*ons_db*/, nullptr, net_type);
CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
LOG_PRINT_L0("Source blockchain storage initialized OK");
log::warning(logcat, "Source blockchain storage initialized OK");
std::vector<crypto::hash> start_txids;
ancestry_state_t state;
fs::path state_file_path = fs::u8path(opt_data_dir) / "ancestry-state.bin";
LOG_PRINT_L0("Loading state data from " << state_file_path);
log::warning(logcat, "Loading state data from {}", state_file_path);
fs::ifstream state_data_in;
state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in);
if (!state_data_in.fail())
@ -480,7 +482,7 @@ int main(int argc, char* argv[])
}
catch (const std::exception &e)
{
MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch");
log::error(logcat, "Failed to load state data from {}, restarting from scratch", state_file_path);
state = ancestry_state_t();
}
state_data_in.close();
@ -494,7 +496,7 @@ int main(int argc, char* argv[])
const uint64_t db_height = db->height();
if (opt_refresh)
{
MINFO("Starting from height " << state.height);
log::info(logcat, "Starting from height {}", state.height);
state.block_cache.reserve(db_height);
for (uint64_t h = state.height; h < db_height; ++h)
{
@ -504,7 +506,7 @@ int main(int argc, char* argv[])
cryptonote::block b;
if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
{
LOG_PRINT_L0("Bad block from db");
log::warning(logcat, "Bad block from db");
return 1;
}
if (opt_cache_blocks)
@ -535,13 +537,13 @@ int main(int argc, char* argv[])
std::string bd;
if (!db->get_pruned_tx_blob(txid, bd))
{
LOG_PRINT_L0("Failed to get txid " << txid << " from db");
log::warning(logcat, "Failed to get txid {} from db", txid);
return 1;
}
cryptonote::transaction tx;
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx))
{
LOG_PRINT_L0("Bad tx: " << txid);
log::warning(logcat, "Bad tx: {}", txid);
return 1;
}
tx_data = ::tx_data_t(tx);
@ -566,7 +568,7 @@ int main(int argc, char* argv[])
crypto::hash output_txid;
if (!get_output_txid(state, db, amount, offset, output_txid))
{
LOG_PRINT_L0("Output originating transaction not found");
log::warning(logcat, "Output originating transaction not found");
return 1;
}
add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid));
@ -575,19 +577,19 @@ int main(int argc, char* argv[])
}
const size_t ancestry_size = get_ancestry(state.ancestry, txid).size();
block_ancestry_size += ancestry_size;
MINFO(txid << ": " << ancestry_size);
log::info(logcat, "{}: {}", txid, ancestry_size);
}
if (!txids.empty())
{
std::string stats_msg;
MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg);
log::info(logcat, "Height {}: {} average over {}{}", h, (block_ancestry_size / txids.size()), txids.size(), stats_msg);
}
state.height = h;
if (stop_requested)
break;
}
LOG_PRINT_L0("Saving state data to " << state_file_path);
log::warning(logcat, "Saving state data to {}", state_file_path);
std::ofstream state_data_out;
state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc);
if (!state_data_out.fail())
@ -599,7 +601,7 @@ int main(int argc, char* argv[])
}
catch (const std::exception &e)
{
MERROR("Failed to save state data to " << state_file_path);
log::error(logcat, "Failed to save state data to {}", state_file_path);
}
state_data_out.close();
}
@ -608,8 +610,8 @@ int main(int argc, char* argv[])
{
if (state.height < db_height)
{
MWARNING("The state file is only built up to height " << state.height << ", but the blockchain reached height " << db_height);
MWARNING("You may want to run with --refresh if you want to get ancestry for newer data");
log::warning(logcat, "The state file is only built up to height {}, but the blockchain reached height {}", state.height, db_height);
log::warning(logcat, "You may want to run with --refresh if you want to get ancestry for newer data");
}
}
@ -622,7 +624,7 @@ int main(int argc, char* argv[])
crypto::hash txid;
if (!get_output_txid(state, db, output_amount, output_offset, txid))
{
LOG_PRINT_L0("Output not found in db");
log::warning(logcat, "Output not found in db");
return 1;
}
start_txids.push_back(txid);
@ -633,7 +635,7 @@ int main(int argc, char* argv[])
cryptonote::block b;
if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
{
LOG_PRINT_L0("Bad block from db");
log::warning(logcat, "Bad block from db");
return 1;
}
for (const crypto::hash &txid: b.tx_hashes)
@ -642,13 +644,13 @@ int main(int argc, char* argv[])
if (start_txids.empty())
{
LOG_PRINT_L0("No transaction(s) to check");
log::warning(logcat, "No transaction(s) to check");
return 1;
}
for (const crypto::hash &start_txid: start_txids)
{
LOG_PRINT_L0("Checking ancestry for txid " << start_txid);
log::warning(logcat, "Checking ancestry for txid {}", start_txid);
std::unordered_map<ancestor, unsigned int> ancestry;
@ -684,22 +686,22 @@ int main(int argc, char* argv[])
crypto::hash output_txid;
if (!get_output_txid(state, db, amount, offset, output_txid))
{
LOG_PRINT_L0("Output originating transaction not found");
log::warning(logcat, "Output originating transaction not found");
return 1;
}
add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid));
txids.push_back(output_txid);
MDEBUG("adding txid: " << output_txid);
log::debug(logcat, "adding txid: {}", output_txid);
}
}
}
}
MINFO("Ancestry for " << start_txid << ": " << get_deduplicated_ancestry(ancestry) << " / " << get_full_ancestry(ancestry));
log::info(logcat, "Ancestry for {}: {} / {}", start_txid, get_deduplicated_ancestry(ancestry), get_full_ancestry(ancestry));
for (const auto &i: ancestry)
{
MINFO(cryptonote::print_money(i.first.amount) << "/" << i.first.offset << ": " << i.second);
log::info(logcat, "{}/{}: {}", cryptonote::print_money(i.first.amount), i.first.offset, i.second);
}
}
@ -707,10 +709,7 @@ done:
core_storage->deinit();
if (opt_show_cache_stats)
MINFO("cache: txes " << std::to_string(cached_txes*100./total_txes)
<< "%, blocks " << std::to_string(cached_blocks*100./total_blocks)
<< "%, outputs " << std::to_string(cached_outputs*100./total_outputs)
<< "%");
log::info(logcat, "cache: txes {}%, blocks {}%, outputs {}%", std::to_string(cached_txes*100./total_txes), std::to_string(cached_blocks*100./total_blocks), std::to_string(cached_outputs*100./total_outputs));
return 0;

117
src/blockchain_utilities/blockchain_blackball.cpp
View File

@ -36,6 +36,7 @@
#include "common/string_util.h"
#include "common/varint.h"
#include "common/file.h"
#include "common/fs-format.h"
#include "common/signal_handler.h"
#include "common/hex.h"
#include "serialization/crypto.h"
@ -47,12 +48,11 @@
#include "version.h"
#include "cryptonote_core/uptime_proof.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "bcutil"
namespace po = boost::program_options;
using namespace cryptonote;
static auto logcat = log::Cat("bcutil");
static const char zerokey[8] = {0};
static const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey };
@ -90,7 +90,7 @@ static bool parse_db_sync_mode(std::string db_sync_mode)
auto options = tools::split_any(db_sync_mode, " :", true);
for(const auto &option : options)
MDEBUG("option: " << option);
log::debug(logcat, "option: {}", option);
// default to fast:async:1
uint64_t DEFAULT_FLAGS = DBF_FAST;
@ -206,14 +206,14 @@ static int resize_env(const char *db_path)
auto si = fs::space(fs::u8path(db_path));
if(si.available < needed)
{
MERROR("!! WARNING: Insufficient free space to extend database !!: " << (si.available / 1000000) << " MB available");
log::error(logcat, "!! WARNING: Insufficient free space to extend database !!: {} MB available", (si.available / 1000000));
return ENOSPC;
}
}
catch(...)
{
// print something but proceed.
MWARNING("Unable to query free disk space.");
log::warning(logcat, "Unable to query free disk space.");
}
mapsize += needed;
@ -227,11 +227,11 @@ static void init(fs::path cache_filename)
bool tx_active = false;
int dbr;
MINFO("Creating spent output cache in " << cache_filename);
log::info(logcat, "Creating spent output cache in {}", cache_filename);
std::error_code ec;
if (fs::create_directories(cache_filename, ec); ec)
MWARNING("Failed to create output cache directory " << cache_filename << ": " << ec.message());
log::warning(logcat, "Failed to create output cache directory {}: {}", cache_filename, ec.message());
int flags = 0;
if (db_flags & DBF_FAST)
@ -392,7 +392,7 @@ static bool for_all_transactions(const fs::path& filename, uint64_t& start_idx,
std::string_view bd{static_cast<const char*>(v.mv_data), v.mv_size};
serialization::parse_binary(bd, tx);
} catch (const std::exception& e) {
LOG_ERROR("Failed to parse transaction from blob: " << e.what());
log::error(logcat, "Failed to parse transaction from blob: {}", e.what());
return false;
}
@ -607,7 +607,7 @@ static std::vector<uint64_t> canonicalize(const std::vector<uint64_t> &v)
}
if (c.size() < v.size())
{
MINFO("Ring has duplicate member(s): " << tools::join(" ", v));
log::info(logcat, "Ring has duplicate member(s): {}", tools::join(" ", v));
}
return c;
}
@ -955,7 +955,7 @@ static void open_db(const fs::path& filename, MDB_env** env, MDB_txn** txn, MDB_
{
std::error_code ec;
if (fs::create_directories(filename, ec); ec)
MWARNING("Failed to create lmdb path " << filename << ": " << ec.message());
log::warning(logcat, "Failed to create lmdb path {}: {}", filename, ec.message());
int flags = MDB_RDONLY;
if (db_flags & DBF_FAST)
@ -967,7 +967,7 @@ static void open_db(const fs::path& filename, MDB_env** env, MDB_txn** txn, MDB_
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
dbr = mdb_env_set_maxdbs(*env, 1);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
MINFO("Opening oxen blockchain at " << filename);
log::info(logcat, "Opening oxen blockchain at {}", filename);
dbr = mdb_env_open(*env, filename.string().c_str(), flags, 0664);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '"
+ filename.u8string() + "': " + std::string(mdb_strerror(dbr)));
@ -1065,7 +1065,7 @@ static std::vector<std::pair<uint64_t, uint64_t>> load_outputs(const fs::path& f
if (!f)
{
MERROR("Failed to load outputs from " << filename << ": " << strerror(errno));
log::error(logcat, "Failed to load outputs from {}: {}", filename, strerror(errno));
return {};
}
while (1)
@ -1073,7 +1073,7 @@ static std::vector<std::pair<uint64_t, uint64_t>> load_outputs(const fs::path& f
char s[256];
if (!fgets(s, sizeof(s), f))
{
MERROR("Error reading from " << filename << ": " << strerror(errno));
log::error(logcat, "Error reading from {}: {}", filename, strerror(errno));
break;
}
if (feof(f))
@ -1091,7 +1091,7 @@ static std::vector<std::pair<uint64_t, uint64_t>> load_outputs(const fs::path& f
}
if (amount == std::numeric_limits<uint64_t>::max())
{
MERROR("Bad format in " << filename);
log::error(logcat, "Bad format in {}", filename);
continue;
}
if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits<uint64_t>::max() - offset)
@ -1105,7 +1105,7 @@ static std::vector<std::pair<uint64_t, uint64_t>> load_outputs(const fs::path& f
}
else
{
MERROR("Bad format in " << filename);
log::error(logcat, "Bad format in {}", filename);
continue;
}
}
@ -1124,7 +1124,7 @@ static bool export_spent_outputs(MDB_cursor* cur, const fs::path& filename)
if (!f)
{
MERROR("Failed to open " << filename << ": " << strerror(errno));
log::error(logcat, "Failed to open {}: {}", filename, strerror(errno));
return false;
}
@ -1141,7 +1141,7 @@ static bool export_spent_outputs(MDB_cursor* cur, const fs::path& filename)
if (dbr)
{
fclose(f);
MERROR("Failed to enumerate spent outputs: " << mdb_strerror(dbr));
log::error(logcat, "Failed to enumerate spent outputs: {}", mdb_strerror(dbr));
return false;
}
const uint64_t amount = *(const uint64_t*)k.mv_data;
@ -1178,9 +1178,6 @@ int main(int argc, char* argv[])
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
uint32_t log_level = 0;
tools::on_startup();
auto opt_size = command_line::boost_option_sizes();
@ -1243,13 +1240,17 @@ int main(int argc, char* argv[])
return 1;
}
mlog_configure(mlog_get_default_log_path("oxen-blockchain-mark-spent-outputs.log"), true);
if (!command_line::is_arg_defaulted(vm, arg_log_level))
mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
LOG_PRINT_L0("Starting...");
auto m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir);
auto log_file_path = m_config_folder + "oxen-blockchain-mark-spent-outputs.log";
log::Level log_level;
if(auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
throw std::runtime_error{"Incorrect log level"};
}
oxen::logging::init(log_file_path, log_level);
log::warning(logcat, "Starting...");
fs::path output_file_path = fs::u8path(command_line::get_arg(vm, arg_blackball_db_dir));
bool opt_rct_only = command_line::get_arg(vm, arg_rct_only);
@ -1265,7 +1266,7 @@ int main(int argc, char* argv[])
std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode);
if (!parse_db_sync_mode(db_sync_mode))
{
MERROR("Invalid db sync mode: " << db_sync_mode);
log::error(logcat, "Invalid db sync mode: {}", db_sync_mode);
return 1;
}
@ -1274,7 +1275,7 @@ int main(int argc, char* argv[])
inputs.push_back(fs::u8path(in));
if (inputs.empty())
{
LOG_PRINT_L0("No inputs given");
log::warning(logcat, "No inputs given");
return 1;
}
@ -1288,7 +1289,7 @@ int main(int argc, char* argv[])
fs::path cache_dir = output_file_path / "spent-outputs-cache";
init(cache_dir);
LOG_PRINT_L0("Scanning for spent outputs...");
log::warning(logcat, "Scanning for spent outputs...");
size_t done = 0;
@ -1317,7 +1318,7 @@ int main(int argc, char* argv[])
{
if (!start_blackballed_outputs)
{
MINFO("Spent outputs database is empty. Either you haven't run the analysis mode yet, or there is really no output marked as spent.");
log::info(logcat, "Spent outputs database is empty. Either you haven't run the analysis mode yet, or there is really no output marked as spent.");
goto skip_secondary_passes;
}
MDB_txn *txn;
@ -1342,7 +1343,7 @@ int main(int argc, char* argv[])
{
uint64_t window_front = (height / STAT_WINDOW - 1) * STAT_WINDOW;
uint64_t window_back = window_front + STAT_WINDOW - 1;
LOG_PRINT_L0(window_front << "-" << window_back << ": " << (100.0f * outs_spent / outs_total) << "% ( " << outs_spent << " / " << outs_total << " )");
log::warning(logcat, "{}-{}: {}% ( {} / {} )", window_front, window_back, (100.0f * outs_spent / outs_total), outs_spent, outs_total);
outs_total = outs_spent = 0;
}
}
@ -1359,11 +1360,11 @@ int main(int argc, char* argv[])
{
uint64_t window_front = (height / STAT_WINDOW) * STAT_WINDOW;
uint64_t window_back = height;
LOG_PRINT_L0(window_front << "-" << window_back << ": " << (100.0f * outs_spent / outs_total) << "% ( " << outs_spent << " / " << outs_total << " )");
log::warning(logcat, "{}-{}: {}% ( {} / {} )", window_front, window_back, (100.0f * outs_spent / outs_total), outs_spent, outs_total);
}
if (stop_requested)
{
MINFO("Stopping scan...");
log::info(logcat, "Stopping scan...");
return false;
}
return true;
@ -1376,7 +1377,7 @@ int main(int argc, char* argv[])
if (!extra_spent_outputs.empty())
{
MINFO("Adding " << extra_spent_outputs.size() << " extra spent outputs");
log::info(logcat, "Adding {} extra spent outputs", extra_spent_outputs.size());
MDB_txn *txn;
int dbr = mdb_txn_begin(env, NULL, 0, &txn);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
@ -1411,9 +1412,9 @@ int main(int argc, char* argv[])
if (n > 0 && start_idx == 0)
{
start_idx = find_first_diverging_transaction(inputs[0], inputs[n]);
LOG_PRINT_L0("First diverging transaction at " << start_idx);
log::warning(logcat, "First diverging transaction at {}", start_idx);
}
LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx);
log::warning(logcat, "Reading blockchain from {} from {}", inputs[n], start_idx);
MDB_txn *txn;
int dbr = mdb_txn_begin(env, NULL, 0, &txn);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
@ -1450,7 +1451,7 @@ int main(int argc, char* argv[])
const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[0]);
if (opt_verbose)
{
MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in a 1-ring");
log::info(logcat, "Marking output {}/{} as spent, due to being used in a 1-ring", output.first, output.second);
std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
}
blackballs.push_back(output);
@ -1464,7 +1465,7 @@ int main(int argc, char* argv[])
const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]);
if (opt_verbose)
{
MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings");
log::info(logcat, "Marking output {}/{} as spent, due to being used in {} identical {}-rings", output.first, output.second, new_ring.size(), new_ring.size());
std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
}
blackballs.push_back(output);
@ -1479,7 +1480,7 @@ int main(int argc, char* argv[])
const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, o);
if (opt_verbose)
{
MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to as many outputs of that amount being spent as exist so far");
log::info(logcat, "Marking output {}/{} as spent, due to as many outputs of that amount being spent as exist so far", output.first, output.second);
std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
}
blackballs.push_back(output);
@ -1494,7 +1495,7 @@ int main(int argc, char* argv[])
const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]);
if (opt_verbose)
{
MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in " << new_ring.size() << " subsets of " << new_ring.size() << "-rings");
log::info(logcat, "Marking output {}/{} as spent, due to being used in {} subsets of {}-rings", output.first, output.second, new_ring.size(), new_ring.size());
std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
}
blackballs.push_back(output);
@ -1504,12 +1505,11 @@ int main(int argc, char* argv[])
}
else if (n > 0 && get_relative_ring(txn, txin.k_image, relative_ring))
{
MDEBUG("Key image " << txin.k_image << " already seen: "
"rings " << tools::join(" ", relative_ring) << ", " << tools::join(" ", txin.key_offsets));
log::debug(logcat, "Key image {} already seen: rings {}, {}", txin.k_image, tools::join(" ", relative_ring), tools::join(" ", txin.key_offsets));
std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
if (relative_ring != txin.key_offsets)
{
MDEBUG("Rings are different");
log::debug(logcat, "Rings are different");
std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(relative_ring);
const std::vector<uint64_t> r1 = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets);
@ -1521,7 +1521,7 @@ int main(int argc, char* argv[])
}
if (common.empty())
{
MERROR("Rings for the same key image are disjoint");
log::error(logcat, "Rings for the same key image are disjoint");
std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
}
else if (common.size() == 1)
@ -1529,7 +1529,7 @@ int main(int argc, char* argv[])
const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, common[0]);
if (opt_verbose)
{
MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in rings with a single common element");
log::info(logcat, "Marking output {}/{} as spent, due to being used in rings with a single common element", output.first, output.second);
std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
}
blackballs.push_back(output);
@ -1538,7 +1538,7 @@ int main(int argc, char* argv[])
}
else
{
MDEBUG("The intersection has more than one element, it's still ok");
log::debug(logcat, "The intersection has more than one element, it's still ok");
std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush;
for (const auto &out: r0)
if (std::find(common.begin(), common.end(), out) != common.end())
@ -1594,7 +1594,7 @@ int main(int argc, char* argv[])
if (stop_requested)
{
MINFO("Stopping scan...");
log::info(logcat, "Stopping scan...");
return false;
}
return true;
@ -1602,7 +1602,7 @@ int main(int argc, char* argv[])
mdb_cursor_close(cur);
dbr = mdb_txn_commit(txn);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr)));
LOG_PRINT_L0("blockchain from " << inputs[n] << " processed till tx idx " << start_idx);
log::warning(logcat, "blockchain from {} processed till tx idx {}", inputs[n], start_idx);
if (stop_requested)
break;
}
@ -1621,7 +1621,7 @@ int main(int argc, char* argv[])
while (!work_spent.empty())
{
LOG_PRINT_L0("Secondary pass on " << work_spent.size() << " spent outputs");
log::warning(logcat, "Secondary pass on {} spent outputs", work_spent.size());
int dbr = resize_env(cache_dir.string().c_str());
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr)));
@ -1659,8 +1659,7 @@ int main(int argc, char* argv[])
const std::pair<uint64_t, uint64_t> output = std::make_pair(od.amount, last_unknown);
if (opt_verbose)
{
MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to being used in a " <<
absolute.size() << "-ring where all other outputs are known to be spent");
log::info(logcat, "Marking output {}/{} as spent, due to being used in a {}-ring where all other outputs are known to be spent", output.first, output.second, absolute.size());
}
blackballs.push_back(output);
if (add_spent_output(cur, output_data(od.amount, last_unknown)))
@ -1671,7 +1670,7 @@ int main(int argc, char* argv[])
if (stop_requested)
{
MINFO("Stopping secondary passes. Secondary passes are not incremental, they will re-run fully.");
log::info(logcat, "Stopping secondary passes. Secondary passes are not incremental, they will re-run fully.");
return 0;
}
}
@ -1687,15 +1686,15 @@ int main(int argc, char* argv[])
skip_secondary_passes:
uint64_t diff = get_num_spent_outputs() - start_blackballed_outputs;
LOG_PRINT_L0(std::to_string(diff) << " new outputs marked as spent, " << get_num_spent_outputs() << " total outputs marked as spent");
log::warning(logcat, "{} new outputs marked as spent, {} total outputs marked as spent", std::to_string(diff), get_num_spent_outputs());
MDB_txn *txn;
dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);
CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
uint64_t pre_rct = 0, rct = 0;
get_num_outputs(txn0, cur0, dbi0, pre_rct, rct);
MINFO("Total pre-rct outputs: " << pre_rct);
MINFO("Total rct outputs: " << rct);
log::info(logcat, "Total pre-rct outputs: {}", pre_rct);
log::info(logcat, "Total rct outputs: {}", rct);
static const struct { const char *key; uint64_t base; } stat_keys[] = {
{ "pre-rct-ring-size-1", pre_rct }, { "rct-ring-size-1", rct },
{ "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct },
@ -1711,7 +1710,7 @@ skip_secondary_passes:
if (!get_stat(txn, key.key, data))
data = 0;
float percent = key.base ? 100.0f * data / key.base : 0.0f;
MINFO(key.key << ": " << data << " (" << percent << "%)");
log::info(logcat, "{}: {} ({}%)", key.key, data, percent);
}
mdb_txn_abort(txn);
@ -1728,7 +1727,7 @@ skip_secondary_passes:
mdb_txn_abort(txn);
}
LOG_PRINT_L0("Blockchain spent output data exported OK");
log::warning(logcat, "Blockchain spent output data exported OK");
close_db(env0, txn0, cur0, dbi0);
close();
return 0;

79
src/blockchain_utilities/blockchain_depth.cpp
View File

@ -29,26 +29,23 @@
#include "common/command_line.h"
#include "common/varint.h"
#include "common/median.h"
#include "common/fs-format.h"
#include "cryptonote_core/cryptonote_core.h"
#include "blockchain_objects.h"
#include "blockchain_db/blockchain_db.h"
#include "cryptonote_core/uptime_proof.h"
#include "version.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "bcutil"
namespace po = boost::program_options;
using namespace cryptonote;
static auto logcat = log::Cat("bcutil");
int main(int argc, char* argv[])
{
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
uint32_t log_level = 0;
tools::on_startup();
auto opt_size = command_line::boost_option_sizes();
@ -90,13 +87,17 @@ int main(int argc, char* argv[])
return 1;
}
mlog_configure(mlog_get_default_log_path("oxen-blockchain-depth.log"), true);
if (!command_line::is_arg_defaulted(vm, arg_log_level))
mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
LOG_PRINT_L0("Starting...");
auto m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir);
auto log_file_path = m_config_folder + "oxen-blockchain-depth.log";
log::Level log_level;
if(auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
throw std::runtime_error{"Incorrect log level"};
}
oxen::logging::init(log_file_path, log_level);
log::warning(logcat, "Starting...");
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
@ -110,7 +111,7 @@ int main(int argc, char* argv[])
std::cerr << "txid and height cannot be given at the same time" << std::endl;
return 1;
}
crypto::hash opt_txid = crypto::null_hash;
crypto::hash opt_txid{};
if (!opt_txid_string.empty())
{
if (!tools::hex_to_type(opt_txid_string, opt_txid))
@ -120,19 +121,19 @@ int main(int argc, char* argv[])
}
}
LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
log::warning(logcat, "Initializing source blockchain (BlockchainDB)");
blockchain_objects_t blockchain_objects = {};
Blockchain *core_storage = &blockchain_objects.m_blockchain;
BlockchainDB *db = new_db();
if (db == NULL)
{
LOG_ERROR("Failed to initialize a database");
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
LOG_PRINT_L0("database: LMDB");
log::warning(logcat, "database: LMDB");
const fs::path filename = fs::u8path(command_line::get_arg(vm, cryptonote::arg_data_dir)) / db->get_db_name();
LOG_PRINT_L0("Loading blockchain from folder " << filename << " ...");
log::warning(logcat, "Loading blockchain from folder {} ...", filename);
try
{
@ -140,13 +141,13 @@ int main(int argc, char* argv[])
}
catch (const std::exception& e)
{
LOG_PRINT_L0("Error opening database: " << e.what());
log::warning(logcat, "Error opening database: {}", e.what());
return 1;
}
r = core_storage->init(db, nullptr /*ons_db*/, nullptr, net_type);
CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
LOG_PRINT_L0("Source blockchain storage initialized OK");
log::warning(logcat, "Source blockchain storage initialized OK");
std::vector<crypto::hash> start_txids;
if (!opt_txid_string.empty())
@ -159,7 +160,7 @@ int main(int argc, char* argv[])
cryptonote::block b;
if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
{
LOG_PRINT_L0("Bad block from db");
log::warning(logcat, "Bad block from db");
return 1;
}
for (const crypto::hash &txid: b.tx_hashes)
@ -170,7 +171,7 @@ int main(int argc, char* argv[])
if (start_txids.empty())
{
LOG_PRINT_L0("No transaction(s) to check");
log::warning(logcat, "No transaction(s) to check");
return 1;
}
@ -180,31 +181,31 @@ int main(int argc, char* argv[])
uint64_t depth = 0;
bool coinbase = false;
LOG_PRINT_L0("Checking depth for txid " << start_txid);
log::warning(logcat, "Checking depth for txid {}", start_txid);
std::vector<crypto::hash> txids(1, start_txid);
while (!coinbase)
{
LOG_PRINT_L0("Considering "<< txids.size() << " transaction(s) at depth " << depth);
log::warning(logcat, "Considering {} transaction(s) at depth {}", txids.size(), depth);
std::vector<crypto::hash> new_txids;
for (const crypto::hash &txid: txids)
{
std::string bd;
if (!db->get_pruned_tx_blob(txid, bd))
{
LOG_PRINT_L0("Failed to get txid " << txid << " from db");
log::warning(logcat, "Failed to get txid {} from db", txid);
return 1;
}
cryptonote::transaction tx;
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx))
{
LOG_PRINT_L0("Bad tx: " << txid);
log::warning(logcat, "Bad tx: {}", txid);
return 1;
}
for (size_t ring = 0; ring < tx.vin.size(); ++ring)
{
if (std::holds_alternative<cryptonote::txin_gen>(tx.vin[ring]))
{
MDEBUG(txid << " is a coinbase transaction");
log::debug(logcat, "{} is a coinbase transaction", txid);
coinbase = true;
goto done;
}
@ -220,7 +221,7 @@ int main(int argc, char* argv[])
cryptonote::block b;
if (!cryptonote::parse_and_validate_block_from_blob(bd, b))
{
LOG_PRINT_L0("Bad block from db");
log::warning(logcat, "Bad block from db");
return 1;
}
// find the tx which created this output
@ -233,13 +234,13 @@ int main(int argc, char* argv[])
{
found = true;
new_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx));
MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx));
log::debug(logcat, "adding txid: {}", cryptonote::get_transaction_hash(b.miner_tx));
break;
}
}
else
{
LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx));
log::warning(logcat, "Bad vout type in txid {}", cryptonote::get_transaction_hash(b.miner_tx));
return 1;
}
}
@ -249,13 +250,13 @@ int main(int argc, char* argv[])
break;
if (!db->get_pruned_tx_blob(block_txid, bd))
{
LOG_PRINT_L0("Failed to get txid " << block_txid << " from db");
log::warning(logcat, "Failed to get txid {} from db", block_txid);
return 1;
}
cryptonote::transaction tx2;
if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2))
{
LOG_PRINT_L0("Bad tx: " << block_txid);
log::warning(logcat, "Bad tx: {}", block_txid);
return 1;
}
for (size_t out = 0; out < tx2.vout.size(); ++out)
@ -266,27 +267,27 @@ int main(int argc, char* argv[])
{
found = true;
new_txids.push_back(block_txid);
MDEBUG("adding txid: " << block_txid);
log::debug(logcat, "adding txid: {}", block_txid);
break;
}
}
else
{
LOG_PRINT_L0("Bad vout type in txid " << block_txid);
log::warning(logcat, "Bad vout type in txid {}", block_txid);
return 1;
}
}
}
if (!found)
{
LOG_PRINT_L0("Output originating transaction not found");
log::warning(logcat, "Output originating transaction not found");
return 1;
}
}
}
else
{
LOG_PRINT_L0("Bad vin type in txid " << txid);
log::warning(logcat, "Bad vin type in txid {}", txid);
return 1;
}
}
@ -298,15 +299,15 @@ int main(int argc, char* argv[])
}
}
done:
LOG_PRINT_L0("Min depth for txid " << start_txid << ": " << depth);
log::warning(logcat, "Min depth for txid {}: {}", start_txid, depth);
depths.push_back(depth);
}
uint64_t cumulative_depth = 0;
for (uint64_t depth: depths)
cumulative_depth += depth;
LOG_PRINT_L0("Average min depth for " << start_txids.size() << " transaction(s): " << cumulative_depth/(float)depths.size());
LOG_PRINT_L0("Median min depth for " << start_txids.size() << " transaction(s): " << tools::median(std::move(depths)));
log::warning(logcat, "Average min depth for {} transaction(s): {}", start_txids.size(), cumulative_depth/(float)depths.size());
log::warning(logcat, "Median min depth for {} transaction(s): {}", start_txids.size(), tools::median(std::move(depths)));
core_storage->deinit();
return 0;

48
src/blockchain_utilities/blockchain_export.cpp
View File

@ -30,28 +30,25 @@
#include "bootstrap_file.h"
#include "blocksdat_file.h"
#include "common/command_line.h"
#include "common/fs-format.h"
#include "cryptonote_core/cryptonote_core.h"
#include "blockchain_objects.h"
#include "version.h"
#include "cryptonote_core/uptime_proof.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "bcutil"
namespace po = boost::program_options;
int main(int argc, char* argv[])
{
using namespace oxen;
auto logcat = log::Cat("bcutil");
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
uint32_t log_level = 0;
uint64_t block_stop = 0;
bool blocks_dat = false;
tools::on_startup();
auto opt_size = command_line::boost_option_sizes();
po::options_description desc_cmd_only("Command line options", opt_size.first, opt_size.second);
@ -92,14 +89,19 @@ int main(int argc, char* argv[])
return 1;
}
mlog_configure(mlog_get_default_log_path("oxen-blockchain-export.log"), true);
if (!command_line::is_arg_defaulted(vm, arg_log_level))
mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
block_stop = command_line::get_arg(vm, arg_block_stop);
LOG_PRINT_L0("Starting...");
auto m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir);
auto log_file_path = m_config_folder + "oxen-blockchain-export.log";
log::Level log_level;
if(auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
throw std::runtime_error{"Incorrect log level"};
}
oxen::logging::init(log_file_path, log_level);
log::warning(logcat, "Starting...");
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
@ -117,42 +119,42 @@ int main(int argc, char* argv[])
output_file_path = fs::u8path(command_line::get_arg(vm, arg_output_file));
else
output_file_path = config_folder / "export" / BLOCKCHAIN_RAW;
LOG_PRINT_L0("Export output file: " << output_file_path.string());
log::warning(logcat, "Export output file: {}", output_file_path.string());
LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
log::warning(logcat, "Initializing source blockchain (BlockchainDB)");
blockchain_objects_t blockchain_objects = {};
Blockchain *core_storage = &blockchain_objects.m_blockchain;
BlockchainDB *db = new_db();
if (db == NULL)
{
LOG_ERROR("Failed to initialize a database");
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
LOG_PRINT_L0("database: LMDB");
log::warning(logcat, "database: LMDB");
auto filename = config_folder / db->get_db_name();
LOG_PRINT_L0("Loading blockchain from folder " << filename << " ...");
log::warning(logcat, "Loading blockchain from folder {} ...", filename);
try
{
db->open(filename, core_storage->nettype(), DBF_RDONLY);
}
catch (const std::exception& e)
{
LOG_PRINT_L0("Error opening database: " << e.what());
log::warning(logcat, "Error opening database: {}", e.what());
return 1;
}
r = core_storage->init(db, nullptr, nullptr, opt_testnet ? cryptonote::network_type::TESTNET : opt_devnet ? cryptonote::network_type::DEVNET : cryptonote::network_type::MAINNET);
if (core_storage->get_blockchain_pruning_seed() && !opt_blocks_dat)
{
LOG_PRINT_L0("Blockchain is pruned, cannot export");
log::warning(logcat, "Blockchain is pruned, cannot export");
return 1;
}
CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
LOG_PRINT_L0("Source blockchain storage initialized OK");
LOG_PRINT_L0("Exporting blockchain raw data...");
log::warning(logcat, "Source blockchain storage initialized OK");
log::warning(logcat, "Exporting blockchain raw data...");
if (opt_blocks_dat)
{
@ -165,7 +167,7 @@ int main(int argc, char* argv[])
r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop);
}
CHECK_AND_ASSERT_MES(r, 1, "Failed to export blockchain raw data");
LOG_PRINT_L0("Blockchain raw data exported OK");
log::warning(logcat, "Blockchain raw data exported OK");
return 0;
CATCH_ENTRY("Export error", 1);

120
src/blockchain_utilities/blockchain_import.cpp
View File

@ -36,6 +36,7 @@
#include <unistd.h>
#include "cryptonote_protocol/quorumnet.h"
#include "epee/misc_log_ex.h"
#include "logging/oxen_logger.h"
#include "bootstrap_file.h"
#include "bootstrap_serialization.h"
#include "blocks/blocks.h"
@ -44,9 +45,8 @@
#include "cryptonote_core/uptime_proof.h"
#include "cryptonote_core/cryptonote_core.h"
#include "common/hex.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "bcutil"
#include "common/fs-format.h"
#include <fmt/color.h>
namespace
{
@ -87,6 +87,8 @@ namespace po = boost::program_options;
using namespace cryptonote;
static auto logcat = log::Cat("bcutil");
// db_mode: safe, fast, fastest
int get_db_flags_from_mode(const std::string& db_mode)
{
@ -142,8 +144,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
cryptonote::block block;
if (!parse_and_validate_block_from_blob(b.block, block))
{
MERROR("Failed to parse block: "
<< tools::type_to_hex(get_blob_hash(b.block)));
log::error(logcat, "Failed to parse block: {}", tools::type_to_hex(get_blob_hash(b.block)));
core.cleanup_handle_incoming_blocks();
return 1;
}
@ -155,12 +156,12 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
std::vector<block> pblocks;
if (!core.prepare_handle_incoming_blocks(blocks, pblocks))
{
MERROR("Failed to prepare to add blocks");
log::error(logcat, "Failed to prepare to add blocks");
return 1;
}
if (!pblocks.empty() && pblocks.size() != blocks.size())
{
MERROR("Unexpected parsed blocks size");
log::error(logcat, "Unexpected parsed blocks size");
core.cleanup_handle_incoming_blocks();
return 1;
}
@ -175,8 +176,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
core.handle_incoming_tx(tx_blob, tvc, tx_pool_options::from_block());
if(tvc.m_verifivation_failed)
{
MERROR("transaction verification failed, tx_id = "
<< tools::type_to_hex(get_blob_hash(tx_blob)));
log::error(logcat, "transaction verification failed, tx_id = {}", tools::type_to_hex(get_blob_hash(tx_blob)));
core.cleanup_handle_incoming_blocks();
return 1;
}
@ -190,14 +190,13 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
if(bvc.m_verifivation_failed)
{
MERROR("Block verification failed, id = "
<< tools::type_to_hex(get_blob_hash(block_entry.block)));
log::error(logcat, "Block verification failed, id = {}", tools::type_to_hex(get_blob_hash(block_entry.block)));
core.cleanup_handle_incoming_blocks();
return 1;
}
if(bvc.m_marked_as_orphaned)
{
MERROR("Block received at sync phase was marked as orphaned");
log::error(logcat, "Block received at sync phase was marked as orphaned");
core.cleanup_handle_incoming_blocks();
return 1;
}
@ -219,7 +218,7 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
if (std::error_code ec; !fs::exists(import_file_path, ec))
{
MFATAL("bootstrap file not found: " << import_file_path);
log::error(logcat, "bootstrap file not found: {}", import_file_path);
return false;
}
@ -232,7 +231,7 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
std::streampos pos;
// BootstrapFile bootstrap(import_file_path);
uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path, pos, seek_height);
MINFO("bootstrap file last block number: " << total_source_blocks-1 << " (zero-based height) total blocks: " << total_source_blocks);
log::info(logcat, "bootstrap file last block number: {} (zero-based height) total blocks: {}", total_source_blocks-1, total_source_blocks);
if (total_source_blocks-1 <= start_height)
{
@ -245,7 +244,7 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
if (import_file.fail())
{
MFATAL("import_file.open() fail");
log::error(logcat, "import_file.open() fail");
return false;
}
@ -269,12 +268,11 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
// These are what we'll try to use, and they don't have to be a determination
// from source and destination blockchains, but those are the defaults.
MINFO("start block: " << start_height << " stop block: " <<
block_stop);
log::info(logcat, "start block: {} stop block: {}", start_height, block_stop);
bool use_batch = opt_batch && !opt_verify;
MINFO("Reading blockchain from bootstrap file...");
log::info(logcat, "Reading blockchain from bootstrap file...");
std::cout << "\n";
std::vector<block_complete_entry> blocks;
@ -313,7 +311,7 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
// TODO: bootstrap.read_chunk();
if (! import_file) {
std::cout << refresh_string;
MINFO("End of file reached");
log::info(logcat, "End of file reached");
quit = 1;
break;
}
@ -324,19 +322,19 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
} catch (const std::exception& e) {
throw std::runtime_error("Error in deserialization of chunk size: "s + e.what());
}
MDEBUG("chunk_size: " << chunk_size);
log::debug(logcat, "chunk_size: {}", chunk_size);
if (chunk_size > BUFFER_SIZE)
{
MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
log::warning(logcat, "WARNING: chunk_size {} > BUFFER_SIZE {}", chunk_size, BUFFER_SIZE);
throw std::runtime_error("Aborting: chunk size exceeds buffer size");
}
if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD)
{
MINFO("NOTE: chunk_size " << chunk_size << " > " << CHUNK_SIZE_WARNING_THRESHOLD);
log::info(logcat, "NOTE: chunk_size {} > {}", chunk_size, CHUNK_SIZE_WARNING_THRESHOLD);
}
else if (chunk_size == 0) {
MFATAL("ERROR: chunk_size == 0");
log::error(logcat, "ERROR: chunk_size == 0");
return 2;
}
import_file.read(buffer_block, chunk_size);
@ -344,26 +342,25 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
if (import_file.eof())
{
std::cout << refresh_string;
MINFO("End of file reached - file was truncated");
log::info(logcat, "End of file reached - file was truncated");
quit = 1;
break;
}
else
{
MFATAL("ERROR: unexpected end of file: bytes read before error: "
<< import_file.gcount() << " of chunk_size " << chunk_size);
log::error(logcat, "ERROR: unexpected end of file: bytes read before error: {} of chunk_size {}", import_file.gcount(), chunk_size);
return 2;
}
}
bytes_read += chunk_size;
MDEBUG("Total bytes read: " << bytes_read);
log::debug(logcat, "Total bytes read: {}", bytes_read);
if (h > block_stop)
{
std::cout << refresh_string << "block " << h-1
<< " / " << block_stop
<< "\n" << std::endl;
MINFO("Specified block number reached - stopping. block: " << h-1 << " total blocks: " << h);
log::info(logcat, "Specified block number reached - stopping. block: {} total blocks: {}", h-1, h);
quit = 1;
break;
}
@ -386,14 +383,14 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
if ((h-1) % display_interval == 0)
{
std::cout << refresh_string;
MDEBUG("loading block number " << h-1);
log::debug(logcat, "loading block number {}", h-1);
}
else
{
MDEBUG("loading block number " << h-1);
log::debug(logcat, "loading block number {}", h-1);
}
b = bp.block;
MDEBUG("block prev_id: " << b.prev_id << "\n");
log::debug(logcat, "block prev_id: {}\n", b.prev_id);
if ((h-1) % progress_interval == 0)
{
@ -458,7 +455,7 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
catch (const std::exception& e)
{
std::cout << refresh_string;
MFATAL("Error adding block to blockchain: " << e.what());
log::error(logcat, "Error adding block to blockchain: {}", e.what());
quit = 2; // make sure we don't commit partial block data
break;
}
@ -488,7 +485,7 @@ int import_from_file(cryptonote::core& core, const fs::path& import_file_path, u
catch (const std::exception& e)
{
std::cout << refresh_string;
MFATAL("exception while reading from file, height=" << h << ": " << e.what());
log::error(logcat, "exception while reading from file, height={}: {}", h, e.what());
return 2;
}
} // while
@ -517,10 +514,10 @@ quitting:
}
core.get_blockchain_storage().get_db().show_stats();
MINFO("Number of blocks imported: " << num_imported);
log::info(logcat, "Number of blocks imported: {}", num_imported);
if (h > 0)
// TODO: if there was an error, the last added block is probably at zero-based height h-2
MINFO("Finished at block: " << h-1 << " total blocks: " << h);
log::info(logcat, "Finished at block: {} total blocks: {}", h-1, h);
std::cout << "\n";
return 0;
@ -532,7 +529,6 @@ int main(int argc, char* argv[])
epee::string_tools::set_module_name_and_folder(argv[0]);
uint32_t log_level = 0;
uint64_t num_blocks = 0;
uint64_t block_stop = 0;
std::string m_config_folder;
@ -638,14 +634,17 @@ int main(int argc, char* argv[])
return 1;
}
m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir);
auto log_file_path = m_config_folder + "oxen-blockchain-import.log";
log::Level log_level;
if(auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
throw std::runtime_error{"Incorrect log level"};
}
mlog_configure(mlog_get_default_log_path("oxen-blockchain-import.log"), true);
if (!command_line::is_arg_defaulted(vm, arg_log_level))
mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
MINFO("Starting...");
oxen::logging::init(log_file_path, log_level);
log::info(logcat, "Starting...");
fs::path import_file_path;
@ -661,32 +660,31 @@ int main(int argc, char* argv[])
return 0;
}
MINFO("database: LMDB");
MINFO("verify: " << std::boolalpha << opt_verify << std::noboolalpha);
log::info(logcat, "database: LMDB");
log::info(logcat, "verify: {}", opt_verify);
if (opt_batch)
{
MINFO("batch: " << std::boolalpha << opt_batch << std::noboolalpha
<< " batch size: " << db_batch_size);
log::info(logcat, "batch: {} batch size: {}", opt_batch, db_batch_size);
}
else
{
MINFO("batch: " << std::boolalpha << opt_batch << std::noboolalpha);
log::info(logcat, "batch: {}", opt_batch);
}
MINFO("resume: " << std::boolalpha << opt_resume << std::noboolalpha);
MINFO("nettype: " << (opt_testnet ? "testnet" : opt_devnet ? "devnet" : "mainnet"));
log::info(logcat, "resume: {}", opt_resume);
log::info(logcat, "nettype: {}", (opt_testnet ? "testnet" : opt_devnet ? "devnet" : "mainnet"));
MINFO("bootstrap file path: " << import_file_path);
MINFO("database path: " << m_config_folder);
log::info(logcat, "bootstrap file path: {}", import_file_path);
log::info(logcat, "database path: {}", m_config_folder);
if (!opt_verify)
{
MCLOG_RED(el::Level::Warning, "global", "\n"
"Import is set to proceed WITHOUT VERIFICATION.\n"
"This is a DANGEROUS operation: if the file was tampered with in transit, or obtained from a malicious source,\n"
"you could end up with a compromised database. It is recommended to NOT use " << arg_noverify.name << ".\n"
"*****************************************************************************************\n"
"You have 90 seconds to press ^C or terminate this program before unverified import starts\n"
"*****************************************************************************************");
log::warning(logcat, fg(fmt::terminal_color::red), "\n\
Import is set to proceed WITHOUT VERIFICATION.\n\
This is a DANGEROUS operation: if the file was tampered with in transit, or obtained from a malicious source,\n\
you could end up with a compromised database. It is recommended to NOT use {}.\n\
*****************************************************************************************\n\
You have 90 seconds to press ^C or terminate this program before unverified import starts\n\
*****************************************************************************************", arg_noverify.name);
sleep(90);
}
@ -714,9 +712,9 @@ int main(int argc, char* argv[])
if (!command_line::is_arg_defaulted(vm, arg_pop_blocks))
{
num_blocks = command_line::get_arg(vm, arg_pop_blocks);
MINFO("height: " << core.get_blockchain_storage().get_current_blockchain_height());
log::info(logcat, "height: {}", core.get_blockchain_storage().get_current_blockchain_height());
pop_blocks(core, num_blocks);
MINFO("height: " << core.get_blockchain_storage().get_current_blockchain_height());
log::info(logcat, "height: {}", core.get_blockchain_storage().get_current_blockchain_height());
return 0;
}

51
src/blockchain_utilities/blockchain_prune.cpp
View File

@ -39,14 +39,13 @@
#include "blockchain_objects.h"
#include "version.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "bcutil"
#define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val}
namespace po = boost::program_options;
using namespace cryptonote;
static auto logcat = log::Cat("bcutil");
static fs::path db_path;
// default to fast:1
@ -57,7 +56,7 @@ static std::error_code replace_file(const fs::path& replacement_name, const fs::
{
std::error_code ec = fs::rename(replacement_name, replaced_name);
if (ec)
MERROR("Error renaming " << replacement_name << " to " << replaced_name << ": " << ec.message());
log::error(logcat, "Error renaming {} to {}: {}", replacement_name, replaced_name, ec.message());
return ec;
}
@ -94,7 +93,7 @@ static void add_size(MDB_env *env, uint64_t bytes)
auto si = fs::space(db_path);
if(si.available < bytes)
{
MERROR("!! WARNING: Insufficient free space to extend database !!: " <<
log::error(logcat, "!! WARNING: Insufficient free space to extend database !!: ",
(si.available >> 20L) << " MB available, " << (bytes >> 20L) << " MB needed");
return;
}
@ -102,7 +101,7 @@ static void add_size(MDB_env *env, uint64_t bytes)
catch(...)
{
// print something but proceed.
MWARNING("Unable to query free disk space.");
log::warning(logcat, "Unable to query free disk space.");
}
MDB_envinfo mei;
@ -117,7 +116,7 @@ static void add_size(MDB_env *env, uint64_t bytes)
if (result)
throw std::runtime_error("Failed to set new mapsize to " + std::to_string(new_mapsize) + ": " + std::string(mdb_strerror(result)));
MGINFO("LMDB Mapsize increased." << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB");
log::info(logcat, "LMDB Mapsize increased. Old: {}MiB, New: {}MiB", mei.me_mapsize / (1024 * 1024), new_mapsize / (1024 * 1024));
}
static void check_resize(MDB_env *env, size_t bytes)
@ -154,7 +153,7 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned
bool tx_active0 = false, tx_active1 = false;
int dbr;
MINFO("Copying " << table);
log::info(logcat, "Copying {}", table);
OXEN_DEFER {
if (tx_active1) mdb_txn_abort(txn1);
@ -251,7 +250,7 @@ static void prune(MDB_env *env0, MDB_env *env1)
bool tx_active0 = false, tx_active1 = false;
int dbr;
MGINFO("Creating pruned txs_prunable");
log::info(logcat, "Creating pruned txs_prunable");
OXEN_DEFER {
if (tx_active1) mdb_txn_abort(txn1);
@ -336,7 +335,7 @@ static void prune(MDB_env *env0, MDB_env *env1)
MDB_val_set(kk, ti->data.tx_id);
if (block_height + PRUNING_TIP_BLOCKS >= blockchain_height)
{
MDEBUG(block_height << "/" << blockchain_height << " is in tip");
log::debug(logcat, "{}/{} is in tip", block_height, blockchain_height);
MDB_val_set(vv, block_height);
dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0);
if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr)));
@ -360,7 +359,7 @@ static void prune(MDB_env *env0, MDB_env *env1)
}
else
{
MDEBUG("" << block_height << "/" << blockchain_height << " should be pruned, dropping");
log::debug(logcat, "{}/{} should be pruned, dropping", block_height, blockchain_height);
}
}
@ -386,7 +385,7 @@ static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags)
auto options = tools::split_any(db_sync_mode, " :", true);
for(const auto &option : options)
MDEBUG("option: " << option);
log::debug(logcat, "option: {}", option);
// default to fast:async:1
uint64_t DEFAULT_FLAGS = DBF_FAST;
@ -490,7 +489,7 @@ int main(int argc, char* argv[])
else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
MINFO("Starting...");
log::info(logcat, "Starting...");
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
@ -504,7 +503,7 @@ int main(int argc, char* argv[])
uint64_t db_flags = 0;
if (!parse_db_sync_mode(db_sync_mode, db_flags))
{
MERROR("Invalid db sync mode: " << db_sync_mode);
log::error(logcat, "Invalid db sync mode: {}", db_sync_mode);
return 1;
}
@ -519,7 +518,7 @@ int main(int argc, char* argv[])
// Blockchain* core_storage = new Blockchain(NULL);
// because unlike blockchain_storage constructor, which takes a pointer to
// tx_memory_pool, Blockchain's constructor takes tx_memory_pool object.
MINFO("Initializing source blockchain (BlockchainDB)");
log::info(logcat, "Initializing source blockchain (BlockchainDB)");
std::array<Blockchain *, 2> core_storage;
fs::path paths[2];
bool already_pruned = false;
@ -531,7 +530,7 @@ int main(int argc, char* argv[])
BlockchainDB* db = new_db();
if (db == NULL)
{
MERROR("Failed to initialize a database");
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
@ -542,7 +541,7 @@ int main(int argc, char* argv[])
{
if (!fs::is_directory(paths[1]))
{
MERROR("LMDB needs a directory path, but a file was passed: " << paths[1].string());
log::error(logcat, "LMDB needs a directory path, but a file was passed: {}", paths[1].string());
return 1;
}
}
@ -550,7 +549,7 @@ int main(int argc, char* argv[])
{
if (!fs::create_directories(paths[1]))
{
MERROR("Failed to create directory: " << paths[1].string());
log::error(logcat, "Failed to create directory: {}", paths[1].string());
return 1;
}
}
@ -561,7 +560,7 @@ int main(int argc, char* argv[])
paths[0] = fs::u8path(data_dir) / db->get_db_name();
}
MINFO("Loading blockchain from folder " << paths[n] << " ...");
log::info(logcat, "Loading blockchain from folder {} ...", paths[n]);
try
{
@ -569,19 +568,19 @@ int main(int argc, char* argv[])
}
catch (const std::exception& e)
{
MERROR("Error opening database: " << e.what());
log::error(logcat, "Error opening database: {}", e.what());
return 1;
}
r = core_storage[n]->init(db, nullptr /*ons_db*/, net_type);
std::string source_dest = n == 0 ? "source" : "pruned";
CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize " << source_dest << " blockchain storage");
MINFO(source_dest << " blockchain storage initialized OK");
log::info(logcat, "{} blockchain storage initialized OK", source_dest);
if (n == 0 && core_storage[0]->get_blockchain_pruning_seed())
{
if (!opt_copy_pruned_database)
{
MERROR("Blockchain is already pruned, use --" << arg_copy_pruned_database.name << " to copy it anyway");
log::error(logcat, "Blockchain is already pruned, use --{} to copy it anyway", arg_copy_pruned_database.name);
return 1;
}
already_pruned = true;
@ -592,7 +591,7 @@ int main(int argc, char* argv[])
core_storage[1]->deinit();
delete core_storage[1];
MINFO("Pruning...");
log::info(logcat, "Pruning...");
MDB_env *env0 = NULL, *env1 = NULL;
open(env0, paths[0], db_flags, true);
open(env1, paths[1], db_flags, false);
@ -624,16 +623,16 @@ int main(int argc, char* argv[])
close(env1);
close(env0);
MINFO("Swapping databases, pre-pruning blockchain will be left in " << paths[0].string() + "-old and can be removed if desired");
log::info(logcat, "Swapping databases, pre-pruning blockchain will be left in {}", paths[0].string() + "-old and can be removed if desired");
fs::path old = paths[0];
old += "-old";
if (replace_file(paths[0], old) || replace_file(paths[1], paths[0]))
{
MERROR("Blockchain pruned OK, but renaming failed");
log::error(logcat, "Blockchain pruned OK, but renaming failed");
return 1;
}
MINFO("Blockchain pruned OK");
log::info(logcat, "Blockchain pruned OK");
return 0;
CATCH_ENTRY("Pruning error", 1);

49
src/blockchain_utilities/blockchain_prune_known_spent_data.cpp
View File

@ -37,12 +37,11 @@
#include "blockchain_db/blockchain_db.h"
#include "version.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "bcutil"
namespace po = boost::program_options;
using namespace cryptonote;
static auto logcat = log::Cat("bcutil");
static std::map<uint64_t, uint64_t> load_outputs(const fs::path& filename)
{
std::map<uint64_t, uint64_t> outputs;
@ -57,7 +56,7 @@ static std::map<uint64_t, uint64_t> load_outputs(const fs::path& filename)
if (!f)
{
MERROR("Failed to load outputs from " << filename << ": " << strerror(errno));
log::error(logcat, "Failed to load outputs from {}: {}", filename, strerror(errno));
return {};
}
while (1)
@ -80,7 +79,7 @@ static std::map<uint64_t, uint64_t> load_outputs(const fs::path& filename)
}
if (amount == std::numeric_limits<uint64_t>::max())
{
MERROR("Bad format in " << filename);
log::error(logcat, "Bad format in {}", filename);
continue;
}
if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits<uint64_t>::max() - offset)
@ -93,7 +92,7 @@ static std::map<uint64_t, uint64_t> load_outputs(const fs::path& filename)
}
else
{
MERROR("Bad format in " << filename);
log::error(logcat, "Bad format in {}", filename);
continue;
}
}
@ -156,7 +155,7 @@ int main(int argc, char* argv[])
else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
LOG_PRINT_L0("Starting...");
log::warning(logcat, "Starting...");
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
@ -166,18 +165,18 @@ int main(int argc, char* argv[])
const auto input = fs::u8path(command_line::get_arg(vm, arg_input));
LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
log::warning(logcat, "Initializing source blockchain (BlockchainDB)");
blockchain_objects_t blockchain_objects = {};
Blockchain *core_storage = &blockchain_objects.m_blockchain;
BlockchainDB *db = new_db();
if (db == NULL)
{
LOG_ERROR("Failed to initialize a database");
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
const fs::path filename = fs::u8path(command_line::get_arg(vm, cryptonote::arg_data_dir)) / db->get_db_name();
LOG_PRINT_L0("Loading blockchain from folder " << filename << " ...");
log::warning(logcat, "Loading blockchain from folder {} ...", filename);
try
{
@ -185,20 +184,20 @@ int main(int argc, char* argv[])
}
catch (const std::exception& e)
{
LOG_PRINT_L0("Error opening database: " << e.what());
log::warning(logcat, "Error opening database: {}", e.what());
return 1;
}
r = core_storage->init(db, nullptr /*ons_db*/, net_type);
CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
LOG_PRINT_L0("Source blockchain storage initialized OK");
log::warning(logcat, "Source blockchain storage initialized OK");
std::map<uint64_t, uint64_t> known_spent_outputs;
if (input.empty())
{
std::map<uint64_t, std::pair<uint64_t, uint64_t>> outputs;
LOG_PRINT_L0("Scanning for known spent data...");
log::warning(logcat, "Scanning for known spent data...");
db->for_all_transactions([&](const crypto::hash &txid, const cryptonote::transaction &tx){
const bool miner_tx = tx.vin.size() == 1 && std::holds_alternative<txin_gen>(tx.vin[0]);
for (const auto &in: tx.vin)
@ -227,11 +226,11 @@ int main(int argc, char* argv[])
}
else
{
LOG_PRINT_L0("Loading known spent data...");
log::warning(logcat, "Loading known spent data...");
known_spent_outputs = load_outputs(input);
}
LOG_PRINT_L0("Pruning known spent data...");
log::warning(logcat, "Pruning known spent data...");
bool stop_requested = false;
tools::signal_handler::install([&stop_requested](int type) {
@ -249,22 +248,22 @@ int main(int argc, char* argv[])
if (i->first == 0)
{
if (opt_verbose)
MINFO("Ignoring output value " << i->first << ", with " << num_outputs << " outputs");
log::info(logcat, "Ignoring output value {}, with {} outputs", i->first, num_outputs);
continue;
}
num_eligible_outputs += num_outputs;
num_eligible_known_spent_outputs += i->second;
if (opt_verbose)
MINFO(i->first << ": " << i->second << "/" << num_outputs);
log::info(logcat, "{}: {}/{}", i->first, i->second, num_outputs);
if (num_outputs > i->second)
continue;
if (num_outputs && num_outputs < i->second)
{
MERROR("More outputs are spent than known for amount " << i->first << ", not touching");
log::error(logcat, "More outputs are spent than known for amount {}, not touching", i->first);
continue;
}
if (opt_verbose)
MINFO("Pruning data for " << num_outputs << " outputs");
log::info(logcat, "Pruning data for {} outputs", num_outputs);
if (!opt_dry_run)
db->prune_outputs(i->first);
num_prunable_outputs += i->second;
@ -272,13 +271,13 @@ int main(int argc, char* argv[])
db->batch_stop();
MINFO("Total outputs: " << num_total_outputs);
MINFO("Known spent outputs: " << num_known_spent_outputs);
MINFO("Eligible outputs: " << num_eligible_outputs);
MINFO("Eligible known spent outputs: " << num_eligible_known_spent_outputs);
MINFO("Prunable outputs: " << num_prunable_outputs);
log::info(logcat, "Total outputs: {}", num_total_outputs);
log::info(logcat, "Known spent outputs: {}", num_known_spent_outputs);
log::info(logcat, "Eligible outputs: {}", num_eligible_outputs);
log::info(logcat, "Eligible known spent outputs: {}", num_eligible_known_spent_outputs);
log::info(logcat, "Prunable outputs: {}", num_prunable_outputs);
LOG_PRINT_L0("Blockchain known spent data pruned OK");
log::warning(logcat, "Blockchain known spent data pruned OK");
core_storage->deinit();
return 0;

47
src/blockchain_utilities/blockchain_stats.cpp
View File

@ -31,6 +31,7 @@
#include "common/command_line.h"
#include "common/varint.h"
#include "common/signal_handler.h"
#include "common/fs-format.h"
#include "cryptonote_basic/cryptonote_boost_serialization.h"
#include "cryptonote_core/cryptonote_core.h"
#include "blockchain_objects.h"
@ -39,9 +40,6 @@
#include "cryptonote_core/uptime_proof.h"
#include <date/date.h>
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "bcutil"
namespace po = boost::program_options;
using namespace cryptonote;
@ -49,14 +47,13 @@ static bool stop_requested = false;
int main(int argc, char* argv[])
{
static auto logcat = log::Cat("bcutil");
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
uint32_t log_level = 0;
uint64_t block_start = 0;
uint64_t block_stop = 0;
tools::on_startup();
auto opt_size = command_line::boost_option_sizes();
@ -104,13 +101,17 @@ int main(int argc, char* argv[])
return 1;
}
mlog_configure(mlog_get_default_log_path("oxen-blockchain-stats.log"), true);
if (!command_line::is_arg_defaulted(vm, arg_log_level))
mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
LOG_PRINT_L0("Starting...");
auto m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir);
auto log_file_path = m_config_folder + "oxen-blockchain-stats.log";
log::Level log_level;
if(auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
throw std::runtime_error{"Incorrect log level"};
}
oxen::logging::init(log_file_path, log_level);
log::warning(logcat, "Starting...");
std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir);
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
@ -123,18 +124,18 @@ int main(int argc, char* argv[])
bool do_ringsize = command_line::get_arg(vm, arg_ringsize);
bool do_hours = command_line::get_arg(vm, arg_hours);
LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
log::warning(logcat, "Initializing source blockchain (BlockchainDB)");
blockchain_objects_t blockchain_objects = {};
Blockchain *core_storage = &blockchain_objects.m_blockchain;
BlockchainDB *db = new_db();
if (db == NULL)
{
LOG_ERROR("Failed to initialize a database");
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
const fs::path filename = fs::u8path(opt_data_dir) / db->get_db_name();
LOG_PRINT_L0("Loading blockchain from folder " << filename << " ...");
log::warning(logcat, "Loading blockchain from folder {} ...", filename);
try
{
@ -142,13 +143,13 @@ int main(int argc, char* argv[])
}
catch (const std::exception& e)
{
LOG_PRINT_L0("Error opening database: " << e.what());
log::warning(logcat, "Error opening database: {}", e.what());
return 1;
}
r = core_storage->init(db, nullptr /*ons_db*/, nullptr, net_type);
CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
LOG_PRINT_L0("Source blockchain storage initialized OK");
log::warning(logcat, "Source blockchain storage initialized OK");
tools::signal_handler::install([](int type) {
stop_requested = true;
@ -157,7 +158,7 @@ int main(int argc, char* argv[])
const uint64_t db_height = db->height();
if (!block_stop)
block_stop = db_height;
MINFO("Starting from height " << block_start << ", stopping at height " << block_stop);
log::info(logcat, "Starting from height {}, stopping at height {}", block_start, block_stop);
/*
* The default output can be plotted with GnuPlot using these commands:
@ -212,7 +213,7 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
cryptonote::block blk;
if (!cryptonote::parse_and_validate_block_from_blob(bd, blk))
{
LOG_PRINT_L0("Bad block from db");
log::warning(logcat, "Bad block from db");
return 1;
}
auto ts = std::chrono::system_clock::from_time_t(blk.timestamp);
@ -261,9 +262,9 @@ skip:
currsz += bd.size();
for (const auto& tx_id : blk.tx_hashes)
{
if (tx_id == crypto::null_hash)
if (!tx_id)
{
throw std::runtime_error("Aborting: tx == null_hash");
throw std::runtime_error("Aborting: null txid");
}
if (!db->get_pruned_tx_blob(tx_id, bd))
{
@ -272,7 +273,7 @@ skip:
transaction tx;
if (!parse_and_validate_tx_base_from_blob(bd, tx))
{
LOG_PRINT_L0("Bad txn from db");
log::warning(logcat, "Bad txn from db");
return 1;
}
currsz += bd.size();

50
src/blockchain_utilities/blockchain_usage.cpp
View File

@ -29,18 +29,18 @@
#include "common/command_line.h"
#include "common/varint.h"
#include "common/fs-format.h"
#include "cryptonote_core/cryptonote_core.h"
#include "blockchain_objects.h"
#include "blockchain_db/blockchain_db.h"
#include "cryptonote_core/uptime_proof.h"
#include "version.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "bcutil"
namespace po = boost::program_options;
using namespace cryptonote;
static auto logcat = log::Cat("quorum_cop");
struct output_data
{
uint64_t amount;
@ -78,11 +78,7 @@ int main(int argc, char* argv[])
TRY_ENTRY();
epee::string_tools::set_module_name_and_folder(argv[0]);
uint32_t log_level = 0;
tools::on_startup();
auto opt_size = command_line::boost_option_sizes();
po::options_description desc_cmd_only("Command line options", opt_size.first, opt_size.second);
@ -122,13 +118,17 @@ int main(int argc, char* argv[])
return 1;
}
mlog_configure(mlog_get_default_log_path("oxen-blockchain-usage.log"), true);
if (!command_line::is_arg_defaulted(vm, arg_log_level))
mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
else
mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
LOG_PRINT_L0("Starting...");
auto m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir);
auto log_file_path = m_config_folder + "oxen-blockchain-usage.log";
log::Level log_level;
if(auto level = oxen::logging::parse_level(command_line::get_arg(vm, arg_log_level).c_str())) {
log_level = *level;
} else {
std::cerr << "Incorrect log level: " << command_line::get_arg(vm, arg_log_level).c_str() << std::endl;
throw std::runtime_error{"Incorrect log level"};
}
oxen::logging::init(log_file_path, log_level);
log::warning(logcat, "Starting...");
bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
bool opt_devnet = command_line::get_arg(vm, cryptonote::arg_devnet_on);
@ -146,7 +146,7 @@ int main(int argc, char* argv[])
// Blockchain* core_storage = new Blockchain(NULL);
// because unlike blockchain_storage constructor, which takes a pointer to
// tx_memory_pool, Blockchain's constructor takes tx_memory_pool object.
LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
log::warning(logcat, "Initializing source blockchain (BlockchainDB)");
const std::string input = command_line::get_arg(vm, arg_input);
blockchain_objects_t blockchain_objects = {};
Blockchain *core_storage = &blockchain_objects.m_blockchain;
@ -154,13 +154,13 @@ int main(int argc, char* argv[])
BlockchainDB *db = new_db();
if (db == NULL)
{
LOG_ERROR("Failed to initialize a database");
log::error(logcat, "Failed to initialize a database");
throw std::runtime_error("Failed to initialize a database");
}
LOG_PRINT_L0("database: LMDB");
log::warning(logcat, "database: LMDB");
const fs::path filename = fs::u8path(input);
LOG_PRINT_L0("Loading blockchain from folder " << filename << " ...");
log::warning(logcat, "Loading blockchain from folder {} ...", filename);
try
{
@ -168,22 +168,22 @@ int main(int argc, char* argv[])
}
catch (const std::exception& e)
{
LOG_PRINT_L0("Error opening database: " << e.what());
log::warning(logcat, "Error opening database: {}", e.what());
return 1;
}
r = core_storage->init(db, nullptr /*ons_db*/, nullptr, net_type);
CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
LOG_PRINT_L0("Source blockchain storage initialized OK");
log::warning(logcat, "Source blockchain storage initialized OK");
LOG_PRINT_L0("Building usage patterns...");
log::warning(logcat, "Building usage patterns...");
size_t done = 0;
std::unordered_map<output_data, std::list<reference>> outputs;
std::unordered_map<uint64_t,uint64_t> indices;
LOG_PRINT_L0("Reading blockchain from " << input);
log::warning(logcat, "Reading blockchain from {}", input);
core_storage->for_all_transactions([&](const crypto::hash &hash, const cryptonote::transaction &tx)->bool
{
const bool coinbase = tx.vin.size() == 1 && std::holds_alternative<txin_gen>(tx.vin[0]);
@ -228,15 +228,15 @@ int main(int argc, char* argv[])
for (const auto &c: counts)
{
float percent = 100.f * c.second / total;
MINFO(std::to_string(c.second) << " outputs used " << c.first << " times (" << percent << "%)");
log::info(logcat, "{} outputs used {} times ({}%)", std::to_string(c.second), c.first, percent);
}
}
else
{
MINFO("No outputs to process");
log::info(logcat, "No outputs to process");
}
LOG_PRINT_L0("Blockchain usage exported OK");
log::warning(logcat, "Blockchain usage exported OK");
return 0;
CATCH_ENTRY("Export error", 1);

29
src/blockchain_utilities/blocksdat_file.cpp
View File

@ -28,16 +28,14 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "blocksdat_file.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "bcutil"
namespace po = boost::program_options;
#include "common/fs-format.h"
using namespace cryptonote;
namespace
{
static auto logcat = log::Cat("bcutil");
std::string refresh_string = "\r \r";
}
@ -52,7 +50,7 @@ bool BlocksdatFile::open_writer(const fs::path& file_path, uint64_t block_stop)
{
if (!fs::is_directory(dir_path))
{
MFATAL("export directory path is a file: " << dir_path);
log::error(logcat, "export directory path is a file: {}", dir_path);
return false;
}
}
@ -60,7 +58,7 @@ bool BlocksdatFile::open_writer(const fs::path& file_path, uint64_t block_stop)
{
if (!fs::create_directory(dir_path))
{
MFATAL("Failed to create directory " << dir_path);
log::error(logcat, "Failed to create directory {}", dir_path);
return false;
}
}
@ -68,7 +66,7 @@ bool BlocksdatFile::open_writer(const fs::path& file_path, uint64_t block_stop)
m_raw_data_file = new std::ofstream();
MINFO("creating file");
log::info(logcat, "creating file");
m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
if (m_raw_data_file->fail())
@ -108,8 +106,7 @@ void BlocksdatFile::write_block(const crypto::hash& block_hash)
crypto::cn_fast_hash(m_hashes.data(), HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash);
memmove(m_hashes.data(), m_hashes.data() + HASH_OF_HASHES_STEP, (m_hashes.size() - HASH_OF_HASHES_STEP) * sizeof(crypto::hash));
m_hashes.resize(m_hashes.size() - HASH_OF_HASHES_STEP);
const std::string data(hash.data, sizeof(hash));
*m_raw_data_file << data;
m_raw_data_file->write(reinterpret_cast<const char*>(hash.data()), hash.size());
}
}
@ -133,21 +130,21 @@ bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
uint64_t block_start = 0;
uint64_t block_stop = 0;
MINFO("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()-1);
log::info(logcat, "source blockchain height: {}", m_blockchain_storage->get_current_blockchain_height()-1);
if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height()))
{
MINFO("Using requested block height: " << requested_block_stop);
log::info(logcat, "Using requested block height: {}", requested_block_stop);
block_stop = requested_block_stop;
}
else
{
block_stop = m_blockchain_storage->get_current_blockchain_height() - 1;
MINFO("Using block height of source blockchain: " << block_stop);
log::info(logcat, "Using block height of source blockchain: {}", block_stop);
}
MINFO("Storing blocks raw data...");
log::info(logcat, "Storing blocks raw data...");
if (!BlocksdatFile::open_writer(output_file, block_stop))
{
MFATAL("failed to open raw file for write");
log::error(logcat, "failed to open raw file for write");
return false;
}
for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
@ -167,7 +164,7 @@ bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
std::cout << refresh_string;
std::cout << "block " << m_cur_height-1 << "/" << block_stop << "\n";
MINFO("Number of blocks exported: " << num_blocks_written);
log::info(logcat, "Number of blocks exported: {}", num_blocks_written);
return BlocksdatFile::close();
}

79
src/blockchain_utilities/bootstrap_file.cpp
View File

@ -32,10 +32,7 @@
#include "bootstrap_file.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "bcutil"
namespace po = boost::program_options;
#include "common/fs-format.h"
using namespace cryptonote;
@ -47,6 +44,7 @@ namespace
const uint32_t header_size = 1024;
std::string refresh_string = "\r \r";
auto logcat = log::Cat("bcutil");
}
@ -60,7 +58,7 @@ bool BootstrapFile::open_writer(const fs::path& file_path)
{
if (!fs::is_directory(dir_path))
{
MFATAL("export directory path is a file: " << dir_path);
log::error(logcat, "export directory path is a file: {}", dir_path);
return false;
}
}
@ -68,7 +66,7 @@ bool BootstrapFile::open_writer(const fs::path& file_path)
{
if (!fs::create_directory(dir_path))
{
MFATAL("Failed to create directory " << dir_path);
log::error(logcat, "Failed to create directory {}", dir_path);
return false;
}
}
@ -81,14 +79,14 @@ bool BootstrapFile::open_writer(const fs::path& file_path)
if (! fs::exists(file_path))
{
MDEBUG("creating file");
log::debug(logcat, "creating file");
do_initialize_file = true;
num_blocks = 0;
}
else
{
num_blocks = count_blocks(file_path.string());
MDEBUG("appending to existing file with height: " << num_blocks-1 << " total blocks: " << num_blocks);
log::debug(logcat, "appending to existing file with height: {} total blocks: {}", num_blocks-1, num_blocks);
}
m_height = num_blocks;
@ -139,7 +137,7 @@ bool BootstrapFile::initialize_file()
uint32_t bd_size = 0;
std::string bd = t_serializable_object_to_blob(bfi);
MDEBUG("bootstrap::file_info size: " << bd.size());
log::debug(logcat, "bootstrap::file_info size: {}", bd.size());
bd_size = bd.size();
try {
@ -151,7 +149,7 @@ bool BootstrapFile::initialize_file()
output_stream_header << bd;
bd = t_serializable_object_to_blob(bbi);
MDEBUG("bootstrap::blocks_info size: " << bd.size());
log::debug(logcat, "bootstrap::blocks_info size: {}", bd.size());
bd_size = bd.size();
try {
@ -175,10 +173,10 @@ void BootstrapFile::flush_chunk()
m_output_stream->flush();
uint32_t chunk_size = m_buffer.size();
// MTRACE("chunk_size " << chunk_size);
// log::trace(logcat, "chunk_size {}", chunk_size);
if (chunk_size > BUFFER_SIZE)
{
MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
log::warning(logcat, "WARNING: chunk_size {} > BUFFER_SIZE {}", chunk_size, BUFFER_SIZE);
}
std::string blob;
@ -200,14 +198,14 @@ void BootstrapFile::flush_chunk()
long num_chars_written = pos_after - pos_before;
if (static_cast<unsigned long>(num_chars_written) != chunk_size)
{
MFATAL("Error writing chunk: height: " << m_cur_height << " chunk_size: " << chunk_size << " num chars written: " << num_chars_written);
log::error(logcat, "Error writing chunk: height: {} chunk_size: {} num chars written: {}", m_cur_height, chunk_size, num_chars_written);
throw std::runtime_error("Error writing chunk");
}
m_buffer.clear();
delete m_output_stream;
m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer);
MDEBUG("flushed chunk: chunk_size: " << chunk_size);
log::debug(logcat, "flushed chunk: chunk_size: {}", chunk_size);
}
void BootstrapFile::write_block(block& block)
@ -223,9 +221,9 @@ void BootstrapFile::write_block(block& block)
// now add all regular transactions
for (const auto& tx_id : block.tx_hashes)
{
if (tx_id == crypto::null_hash)
if (!tx_id)
{
throw std::runtime_error("Aborting: tx == null_hash");
throw std::runtime_error("Aborting: null txid");
}
transaction tx = m_blockchain_storage->get_db().get_tx(tx_id);
@ -271,10 +269,10 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
m_blockchain_storage = _blockchain_storage;
m_tx_pool = _tx_pool;
uint64_t progress_interval = 100;
MINFO("Storing blocks raw data...");
log::info(logcat, "Storing blocks raw data...");
if (!BootstrapFile::open_writer(output_file))
{
MFATAL("failed to open raw file for write");
log::error(logcat, "failed to open raw file for write");
return false;
}
block b;
@ -284,16 +282,16 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
// height.
uint64_t block_start = m_height;
uint64_t block_stop = 0;
MINFO("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()-1);
log::info(logcat, "source blockchain height: {}", m_blockchain_storage->get_current_blockchain_height()-1);
if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height()))
{
MINFO("Using requested block height: " << requested_block_stop);
log::info(logcat, "Using requested block height: {}", requested_block_stop);
block_stop = requested_block_stop;
}
else
{
block_stop = m_blockchain_storage->get_current_blockchain_height() - 1;
MINFO("Using block height of source blockchain: " << block_stop);
log::info(logcat, "Using block height of source blockchain: {}", block_stop);
}
for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
{
@ -319,9 +317,9 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
std::cout << refresh_string;
std::cout << "block " << m_cur_height-1 << "/" << block_stop << "\n";
MINFO("Number of blocks exported: " << num_blocks_written);
log::info(logcat, "Number of blocks exported: {}", num_blocks_written);
if (num_blocks_written > 0)
MINFO("Largest chunk: " << m_max_chunk << " bytes");
log::info(logcat, "Largest chunk: {} bytes", m_max_chunk);
return BootstrapFile::close();
}
@ -345,11 +343,11 @@ uint64_t BootstrapFile::seek_to_first_chunk(fs::ifstream& import_file)
if (file_magic != blockchain_raw_magic)
{
MFATAL("bootstrap file not recognized");
log::error(logcat, "bootstrap file not recognized");
throw std::runtime_error("Aborting");
}
else
MINFO("bootstrap file recognized");
log::info(logcat, "bootstrap file recognized");
uint32_t buflen_file_info;
@ -362,7 +360,7 @@ uint64_t BootstrapFile::seek_to_first_chunk(fs::ifstream& import_file)
} catch (const std::exception& e) {
throw std::runtime_error("Error in deserialization of buflen_file_info: "s + e.what());
}
MINFO("bootstrap::file_info size: " << buflen_file_info);
log::info(logcat, "bootstrap::file_info size: {}", buflen_file_info);
if (buflen_file_info > sizeof(buf1))
throw std::runtime_error("Error: bootstrap::file_info size exceeds buffer size");
@ -376,9 +374,9 @@ uint64_t BootstrapFile::seek_to_first_chunk(fs::ifstream& import_file)
} catch (const std::exception& e) {
throw std::runtime_error("Error in deserialization of bootstrap::file_info: "s + e.what());
}
MINFO("bootstrap file v" << unsigned(bfi.major_version) << "." << unsigned(bfi.minor_version));
MINFO("bootstrap magic size: " << sizeof(file_magic));
MINFO("bootstrap header size: " << bfi.header_size);
log::info(logcat, "bootstrap file v{}.{}", unsigned(bfi.major_version), unsigned(bfi.minor_version));
log::info(logcat, "bootstrap magic size: {}", sizeof(file_magic));
log::info(logcat, "bootstrap header size: {}", bfi.header_size);
uint64_t full_header_size = sizeof(file_magic) + bfi.header_size;
import_file.seekg(full_header_size);
@ -398,7 +396,7 @@ uint64_t BootstrapFile::count_bytes(fs::ifstream& import_file, uint64_t blocks,
import_file.read(buf1, sizeof(chunk_size));
if (!import_file) {
std::cout << refresh_string;
MDEBUG("End of file reached");
log::debug(logcat, "End of file reached");
quit = true;
break;
}
@ -409,32 +407,29 @@ uint64_t BootstrapFile::count_bytes(fs::ifstream& import_file, uint64_t blocks,
} catch (const std::exception& e) {
throw std::runtime_error("Error in deserialization of chunk_size: "s + e.what());
}
MDEBUG("chunk_size: " << chunk_size);
log::debug(logcat, "chunk_size: {}", chunk_size);
if (chunk_size > BUFFER_SIZE)
{
std::cout << refresh_string;
MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE
<< " height: " << h-1 << ", offset " << bytes_read);
log::warning(logcat, "WARNING: chunk_size {} > BUFFER_SIZE {} height: {}, offset {}", chunk_size, BUFFER_SIZE, h-1, bytes_read);
throw std::runtime_error("Aborting: chunk size exceeds buffer size");
}
if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD)
{
std::cout << refresh_string;
MDEBUG("NOTE: chunk_size " << chunk_size << " > " << CHUNK_SIZE_WARNING_THRESHOLD << " << height: "
<< h-1 << ", offset " << bytes_read);
log::debug(logcat, "NOTE: chunk_size {} > {} height: {}, offset {}", chunk_size, CHUNK_SIZE_WARNING_THRESHOLD, h-1, bytes_read);
}
else if (chunk_size <= 0) {
std::cout << refresh_string;
MDEBUG("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h-1 << ", offset " << bytes_read);
log::debug(logcat, "ERROR: chunk_size {} <= 0 height: {}, offset {}", chunk_size, h-1, bytes_read);
throw std::runtime_error("Aborting");
}
// skip to next expected block size value
import_file.seekg(chunk_size, std::ios_base::cur);
if (! import_file) {
std::cout << refresh_string;
MFATAL("ERROR: unexpected end of file: bytes read before error: "
<< import_file.gcount() << " of chunk_size " << chunk_size);
log::error(logcat, "ERROR: unexpected end of file: bytes read before error: {} of chunk_size {}", import_file.gcount(), chunk_size);
throw std::runtime_error("Aborting");
}
bytes_read += chunk_size;
@ -459,7 +454,7 @@ uint64_t BootstrapFile::count_blocks(const fs::path& import_file_path, std::stre
{
if (std::error_code ec; !fs::exists(import_file_path, ec))
{
MFATAL("bootstrap file not found: " << import_file_path);
log::error(logcat, "bootstrap file not found: {}", import_file_path);
throw std::runtime_error("Aborting");
}
fs::ifstream import_file{import_file_path, std::ios::binary};
@ -468,14 +463,14 @@ uint64_t BootstrapFile::count_blocks(const fs::path& import_file_path, std::stre
uint64_t h = 0;
if (import_file.fail())
{
MFATAL("import_file.open() fail");
log::error(logcat, "import_file.open() fail");
throw std::runtime_error("Aborting");
}
uint64_t full_header_size; // 4 byte magic + length of header structures
full_header_size = seek_to_first_chunk(import_file);
MINFO("Scanning blockchain from bootstrap file...");
log::info(logcat, "Scanning blockchain from bootstrap file...");
bool quit = false;
uint64_t bytes_read = 0, blocks;
int progress_interval = 10;
@ -495,7 +490,7 @@ uint64_t BootstrapFile::count_blocks(const fs::path& import_file_path, std::stre
std::flush;
// std::cout << refresh_string;
MDEBUG("Number bytes scanned: " << bytes_read);
log::debug(logcat, "Number bytes scanned: {}", bytes_read);
}
import_file.close();

1
src/checkpoints/CMakeLists.txt
View File

@ -38,4 +38,5 @@ target_link_libraries(checkpoints
Boost::program_options
Boost::serialization
filesystem
logging
extra)

29
src/checkpoints/checkpoints.cpp
View File

@ -41,18 +41,21 @@
#include "common/oxen.h"
#include "common/file.h"
#include "common/fs-format.h"
#include "common/hex.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "checkpoints"
namespace cryptonote
{
static auto logcat = log::Cat("checkpoints");
bool checkpoint_t::check(crypto::hash const &hash) const
{
bool result = block_hash == hash;
if (result) MINFO ("CHECKPOINT PASSED FOR HEIGHT " << height << " " << block_hash);
else MWARNING("CHECKPOINT FAILED FOR HEIGHT " << height << ". EXPECTED HASH " << block_hash << "GIVEN HASH: " << hash);
if (result)
log::info(logcat, "CHECKPOINT PASSED FOR HEIGHT {} {}", height, block_hash);
else
log::warning(logcat, "CHECKPOINT FAILED FOR HEIGHT {}. EXPECTED HASH {}GIVEN HASH: {}", height, block_hash, hash);
return result;
};
@ -73,7 +76,7 @@ namespace cryptonote
crypto::hash get_newest_hardcoded_checkpoint(cryptonote::network_type nettype, uint64_t *height)
{
crypto::hash result = crypto::null_hash;
crypto::hash result{};
*height = 0;
if (nettype != network_type::MAINNET && nettype != network_type::TESTNET)
return result;
@ -93,7 +96,7 @@ namespace cryptonote
{
if (std::error_code ec; !fs::exists(json_hashfile_fullpath, ec))
{
LOG_PRINT_L1("Blockchain checkpoints file not found");
log::info(logcat, "Blockchain checkpoints file not found");
return true;
}
@ -102,7 +105,7 @@ namespace cryptonote
!tools::slurp_file(json_hashfile_fullpath, contents) ||
!epee::serialization::load_t_from_json(hashes, contents))
{
MERROR("Error loading checkpoints from " << json_hashfile_fullpath);
log::error(logcat, "Error loading checkpoints from {}", json_hashfile_fullpath);
return false;
}
@ -119,14 +122,14 @@ namespace cryptonote
}
catch (const std::exception &e)
{
MERROR("Get block checkpoint from DB failed at height: " << height << ", what = " << e.what());
log::error(logcat, "Get block checkpoint from DB failed at height: {}, what = {}", height, e.what());
return false;
}
}
//---------------------------------------------------------------------------
bool checkpoints::add_checkpoint(uint64_t height, const std::string& hash_str)
{
crypto::hash h = crypto::null_hash;
crypto::hash h{};
bool r = tools::hex_to_type(hash_str, h);
CHECK_AND_ASSERT_MES(r, false, "Failed to parse checkpoint hash string into binary representation!");
@ -158,7 +161,7 @@ namespace cryptonote
}
catch (const std::exception& e)
{
MERROR("Failed to add checkpoint with hash: " << checkpoint.block_hash << " at height: " << checkpoint.height << ", what = " << e.what());
log::error(logcat, "Failed to add checkpoint with hash: {} at height: {}, what = {}", checkpoint.block_hash, checkpoint.height, e.what());
result = false;
}
@ -199,7 +202,7 @@ namespace cryptonote
}
catch (const std::exception &e)
{
MERROR("Pruning block checkpoint on block added failed non-trivially at height: " << m_last_cull_height << ", what = " << e.what());
log::error(logcat, "Pruning block checkpoint on block added failed non-trivially at height: {}, what = {}", m_last_cull_height, e.what());
}
}
@ -226,7 +229,7 @@ namespace cryptonote
}
catch (const std::exception &e)
{
MERROR("Remove block checkpoint on detach failed non-trivially at height: " << delete_height << ", what = " << e.what());
log::error(logcat, "Remove block checkpoint on detach failed non-trivially at height: {}, what = {}", delete_height, e.what());
}
}
}

1
src/checkpoints/checkpoints.h
View File

@ -31,7 +31,6 @@
#pragma once
#include <vector>
#include "epee/misc_log_ex.h"
#include "crypto/hash.h"
#include "cryptonote_config.h"
#include "cryptonote_core/service_node_voting.h"

4
src/common/CMakeLists.txt
View File

@ -39,7 +39,6 @@ add_library(common
oxen.cpp
notify.cpp
password.cpp
perf_timer.cpp
pruning.cpp
random.cpp
rules.cpp
@ -65,9 +64,10 @@ target_link_libraries(common
cncrypto
oxenmq::oxenmq
filesystem
oxen::logging
fmt::fmt
date::date
fmt::fmt
PRIVATE
sodium
logging
extra)

16
src/common/apply_permutation.h
View File

@ -34,7 +34,7 @@
#include <vector>
#include <functional>
#include "epee/misc_log_ex.h"
#include "logging/oxen_logger.h"
namespace tools
{
@ -44,7 +44,12 @@ void apply_permutation(std::vector<size_t> permutation, const F &swap)
{
//sanity check
for (size_t n = 0; n < permutation.size(); ++n)
CHECK_AND_ASSERT_THROW_MES(std::find(permutation.begin(), permutation.end(), n) != permutation.end(), "Bad permutation");
if (std::find(permutation.begin(), permutation.end(), n) == permutation.end())
{
log::error(globallogcat, "Bad permutation");
throw std::runtime_error("Bad permutation");
return;
}
for (size_t i = 0; i < permutation.size(); ++i)
{
@ -63,7 +68,12 @@ void apply_permutation(std::vector<size_t> permutation, const F &swap)
template<typename T>
void apply_permutation(const std::vector<size_t> &permutation, std::vector<T> &v)
{
CHECK_AND_ASSERT_THROW_MES(permutation.size() == v.size(), "Mismatched vector sizes");
if (permutation.size() != v.size())
{
log::error(globallogcat, "Mismatched vector sizes");
throw std::runtime_error("Mismatched vector sizes");
return;
}
apply_permutation(permutation, [&v](size_t i0, size_t i1){ std::swap(v[i0], v[i1]); });
}

3
src/common/boost_serialization_helper.h
View File

@ -30,11 +30,12 @@
#pragma once
#include "epee/misc_log_ex.h"
#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/portable_binary_oarchive.hpp>
#include <boost/archive/portable_binary_iarchive.hpp>
#include "epee/misc_log_ex.h"
#include "fs.h"
#include <fstream>
namespace tools

22
src/common/command_line.cpp
View File

@ -34,6 +34,10 @@
#ifdef HAVE_READLINE
# include "epee/readline_buffer.h"
#endif
#include <iostream>
#ifdef _WIN32
#include "windows.h"
#endif
namespace command_line
{
@ -104,5 +108,23 @@ void clear_screen()
#endif
}
bool handle_error_helper(const boost::program_options::options_description& desc, std::function<bool()> parser) {
try
{
return parser();
}
catch (const std::exception& e)
{
std::cerr << "Failed to parse arguments: " << e.what() << std::endl;
std::cerr << desc << std::endl;
return false;
}
catch (...)
{
std::cerr << "Failed to parse arguments: unknown exception" << std::endl;
std::cerr << desc << std::endl;
return false;
}
}
}

76
src/common/command_line.h
View File

@ -31,19 +31,21 @@
#pragma once
#include <functional>
#include <sstream>
#include <array>
#include <type_traits>
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/variables_map.hpp>
#include "epee/misc_log_ex.h"
#include "common/string_util.h"
#include "common/i18n.h"
#include "common/format.h"
#include "logging/oxen_logger.h"
namespace command_line
{
namespace log = oxen::log;
inline const char* tr(const char* str) { return i18n_translate(str, "command_line"); }
/// @return True if `str` is (case-insensitively) y, yes, a potentially translated yes, or any of
@ -137,24 +139,12 @@ namespace command_line
namespace {
template <typename T>
struct arg_stringify {
const T& v;
arg_stringify(const T& val) : v{val} {}
};
template <typename T>
std::ostream& operator<<(std::ostream& o, const arg_stringify<T>& a) {
return o << a.v;
std::string arg_stringify(const T& a) {
return "{}"_format(a);
}
template <typename T>
std::ostream& operator<<(std::ostream& o, const arg_stringify<std::vector<T>>& a) {
o << '{';
bool first = true;
for (auto& x : a.v) {
if (first) first = false;
else o << ",";
o << x;
}
return o << '}';
std::string arg_stringify(const std::vector<T>& v) {
return "{{{}}}"_format(fmt::join(v, ","));
}
}
@ -163,11 +153,11 @@ namespace command_line
{
auto semantic = boost::program_options::value<T>();
if (!arg.not_use_default) {
std::ostringstream format;
format << arg_stringify{arg.depf(false, true, arg.default_value)} << ", "
<< arg_stringify{arg.depf(true, true, arg.default_value)} << " if '"
<< arg.ref.name << "'";
semantic->default_value(arg.depf(arg.ref.default_value, true, arg.default_value), format.str());
auto default_display = "{}, {} if '{}'"_format(
arg_stringify(arg.depf(false, true, arg.default_value)),
arg_stringify(arg.depf(true, true, arg.default_value)),
arg.ref.name);
semantic->default_value(arg.depf(arg.ref.default_value, true, arg.default_value), default_display);
}
return semantic;
}
@ -179,17 +169,19 @@ namespace command_line
if (!arg.not_use_default) {
std::array<bool, NUM_DEPS> depval;
depval.fill(false);
std::ostringstream format;
format << arg_stringify{arg.depf(depval, true, arg.default_value)};
auto default_display = arg_stringify(arg.depf(depval, true, arg.default_value));
for (size_t i = 0; i < depval.size(); ++i)
{
depval.fill(false);
depval[i] = true;
format << ", " << arg_stringify{arg.depf(depval, true, arg.default_value)} << " if '" << arg.ref[i]->name << "'";
fmt::format_to(std::back_inserter(default_display),
", {} if '{}'",
arg_stringify(arg.depf(depval, true, arg.default_value)),
arg.ref[i]->name);
}
for (size_t i = 0; i < depval.size(); ++i)
depval[i] = arg.ref[i]->default_value;
semantic->default_value(arg.depf(depval, true, arg.default_value), format.str());
semantic->default_value(arg.depf(depval, true, arg.default_value), default_display);
}
return semantic;
}
@ -216,7 +208,8 @@ namespace command_line
{
if (0 != description.find_nothrow(arg.name, false))
{
CHECK_AND_ASSERT_MES(!unique, void(), "Argument already exists: " << arg.name);
if (!unique)
log::error(globallogcat, "Argument already exists: {}", arg.name);
return;
}
@ -228,7 +221,8 @@ namespace command_line
{
if (0 != description.find_nothrow(arg.name, false))
{
CHECK_AND_ASSERT_MES(!unique, void(), "Argument already exists: " << arg.name);
if (!unique)
log::error(globallogcat, "Argument already exists: {}", arg.name);
return;
}
@ -240,7 +234,8 @@ namespace command_line
{
if (0 != description.find_nothrow(arg.name, false))
{
CHECK_AND_ASSERT_MES(!unique, void(), "Argument already exists: " << arg.name);
if (!unique)
log::error(globallogcat, "Argument already exists: {}", arg.name);
return;
}
@ -260,26 +255,7 @@ namespace command_line
return parser.run();
}
template<typename F>
bool handle_error_helper(const boost::program_options::options_description& desc, F parser)
{
try
{
return parser();
}
catch (const std::exception& e)
{
std::cerr << "Failed to parse arguments: " << e.what() << std::endl;
std::cerr << desc << std::endl;
return false;
}
catch (...)
{
std::cerr << "Failed to parse arguments: unknown exception" << std::endl;
std::cerr << desc << std::endl;
return false;
}
}
bool handle_error_helper(const boost::program_options::options_description& desc, std::function<bool()> parser);
template<typename T, bool required, bool dependent, int NUM_DEPS>
std::enable_if_t<!std::is_same_v<T, bool>, bool> has_arg(const boost::program_options::variables_map& vm, const arg_descriptor<T, required, dependent, NUM_DEPS>& arg)

6
src/common/expect.cpp
View File

@ -26,8 +26,8 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "expect.h"
#include "common/fs.h"
#include <easylogging++.h>
#include <string>
namespace detail
@ -48,9 +48,7 @@ namespace detail
error_msg.append("thrown at ");
// remove path, get just filename + extension
char buff[256] = {0};
el::base::utils::File::buildBaseFilename(file, buff, sizeof(buff) - 1);
error_msg.append(buff);
error_msg.append(fs::path(file).filename().string());
error_msg.push_back(':');
error_msg.append(std::to_string(line));

35
src/common/file.cpp
View File

@ -30,9 +30,11 @@
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
//
#include "file.h"
#include "epee/misc_log_ex.h"
#include "fs-format.h"
#include "logging/oxen_logger.h"
#include <unistd.h>
#include <cstdio>
#include <fstream>
#ifdef WIN32
#include "epee/string_tools.h"
@ -59,11 +61,10 @@
#include "cryptonote_config.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "util"
namespace tools {
static auto logcat = log::Cat("util");
#ifndef _WIN32
static int flock_exnb(int fd)
{
@ -77,7 +78,7 @@ namespace tools {
fl.l_len = 0;
ret = fcntl(fd, F_SETLK, &fl);
if (ret < 0)
MERROR("Error locking fd " << fd << ": " << errno << " (" << strerror(errno) << ")");
log::error(logcat, "Error locking fd {}: {} ({})", fd, errno, strerror(errno));
return ret;
}
#endif
@ -207,14 +208,14 @@ namespace tools {
memset(&ov, 0, sizeof(ov));
if (!LockFileEx(m_fd, LOCKFILE_FAIL_IMMEDIATELY | LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &ov))
{
MERROR("Failed to lock " << filename << ": " << std::error_code(GetLastError(), std::system_category()));
log::error(logcat, "Failed to lock {}: {}", filename, std::error_code(GetLastError(), std::system_category()).message());
CloseHandle(m_fd);
m_fd = INVALID_HANDLE_VALUE;
}
}
else
{
MERROR("Failed to open " << filename << ": " << std::error_code(GetLastError(), std::system_category()));
log::error(logcat, "Failed to open {}: {}", filename, std::error_code(GetLastError(), std::system_category()).message());
}
#else
m_fd = open(filename.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666);
@ -222,14 +223,14 @@ namespace tools {
{
if (flock_exnb(m_fd) == -1)
{
MERROR("Failed to lock " << filename << ": " << std::strerror(errno));
log::error(logcat, "Failed to lock {}: {}", filename, std::strerror(errno));
close(m_fd);
m_fd = -1;
}
}
else
{
MERROR("Failed to open " << filename << ": " << std::strerror(errno));
log::error(logcat, "Failed to open {}: {}", filename, std::strerror(errno));
}
#endif
}
@ -264,7 +265,7 @@ namespace tools {
return fs::path{psz_path};
}
LOG_ERROR("SHGetSpecialFolderPathW() failed, could not obtain requested path.");
log::error(logcat, "SHGetSpecialFolderPathW() failed, could not obtain requested path.");
return "";
}
#endif
@ -302,17 +303,11 @@ namespace tools {
bool slurp_file(const fs::path& filename, std::string& contents)
{
fs::ifstream in;
in.exceptions(std::ifstream::failbit | std::ifstream::badbit);
try {
in.open(filename, std::ios::binary | std::ios::in | std::ios::ate);
contents.clear();
contents.resize(in.tellg());
in.seekg(0);
in.read(contents.data(), contents.size());
auto bytes_read = in.gcount();
if (static_cast<size_t>(bytes_read) < contents.size())
contents.resize(bytes_read);
fs::ifstream in(filename, std::ios::binary);
std::string content((std::istreambuf_iterator<char>(in)), (std::istreambuf_iterator<char>()));
contents = std::move(content);
return true;
} catch (...) {
return false;

6
src/common/format.h Normal file
View File

@ -0,0 +1,6 @@
#pragma once
#include <oxen/log/format.hpp>
// Make ""_format available globally:
using namespace oxen::log::literals;

117
src/common/formattable.h Normal file
View File

@ -0,0 +1,117 @@
#pragma once
#include <fmt/format.h>
#include <string_view>
#include <type_traits>
#include <utility>
namespace formattable {
// Types can opt-in to being formattable as a string by specializing this to true. Such a type
// must have one of:
//
// - a `to_string()` method on the type; when formatted we will call `val.to_string()` to format
// it as a string.
// - a `to_string(val)` function in the same namespace as the type; we will call it to format it
// as a string.
//
// The function should return something string-like (string, string_view, const char*).
//
// For instance to opt-in MyType for such string formatting, use:
//
// template <> inline constexpr bool formattable::via_to_string<MyType> = true;
//
// You can also partially specialize; for instance to make all derived classes of a common base
// type formattable via to_string you could do:
//
// template <T> inline constexpr bool formattable::via_to_string<T,
// std::enable_if_t<std::is_base_of_v<MyBaseType, T>>> = true;
//
template <typename T, typename SFINAE = void>
constexpr bool via_to_string = false;
// Same as above, but looks for a to_hex_string() instead of to_string(), for types that get
// dumped as hex.
template <typename T, typename SFINAE = void>
constexpr bool via_to_hex_string = false;
// Scoped enums can alternatively be formatted as their underlying integer value by specializing
// this function to true:
template <typename T, typename SFINAE = void>
constexpr bool via_underlying = false;
namespace detail {
template <typename T, typename SFINAE = void>
constexpr bool has_to_string_method = false;
template <typename T, typename SFINAE = void>
constexpr bool has_to_hex_string_method = false;
template <typename T>
inline constexpr bool has_to_string_method<T,
std::void_t<decltype(std::declval<const T&>().to_string())>
> = true;
template <typename T>
inline constexpr bool has_to_hex_string_method<T,
std::void_t<decltype(std::declval<const T&>().to_hex_string())>
> = true;
} // namespace detail
template <typename T>
struct to_string_formatter : fmt::formatter<std::string_view> {
template <typename FormatContext>
auto format(const T& val, FormatContext& ctx) const {
if constexpr (::formattable::detail::has_to_string_method<T>)
return formatter<std::string_view>::format(val.to_string(), ctx);
else
return formatter<std::string_view>::format(to_string(val), ctx);
}
};
template <typename T>
struct to_hex_string_formatter : fmt::formatter<std::string_view> {
template <typename FormatContext>
auto format(const T& val, FormatContext& ctx) const {
if constexpr (::formattable::detail::has_to_hex_string_method<T>)
return formatter<std::string_view>::format(val.to_hex_string(), ctx);
else
return formatter<std::string_view>::format(to_hex_string(val), ctx);
}
};
template <typename T>
struct underlying_t_formatter : fmt::formatter<std::underlying_type_t<T>> {
#ifdef __cpp_lib_is_scoped_enum // C++23
static_assert(std::is_scoped_enum_v<T>);
#else
static_assert(std::is_enum_v<T> && !std::is_convertible_v<T, std::underlying_type_t<T>>,
"formattable::via_underlying<T> type is not a scoped enum");
#endif
template <typename FormatContext>
auto format(const T& val, FormatContext& ctx) const {
using Underlying = std::underlying_type_t<T>;
return fmt::formatter<Underlying>::format(static_cast<Underlying>(val), ctx);
}
};
} // namespace formattable
namespace fmt {
template <typename T, typename Char>
struct formatter<T, Char, std::enable_if_t<::formattable::via_to_string<T>>>
: ::formattable::to_string_formatter<T> {};
template <typename T, typename Char>
struct formatter<T, Char, std::enable_if_t<::formattable::via_to_hex_string<T>>>
: ::formattable::to_hex_string_formatter<T> {};
template <typename T, typename Char>
struct formatter<T, Char, std::enable_if_t<::formattable::via_underlying<T>>>
: ::formattable::underlying_t_formatter<T> {};
} // namespace fmt

20
src/common/fs-format.h Normal file
View File

@ -0,0 +1,20 @@
#pragma once
#include "fs.h"
#ifndef USE_GHC_FILESYSTEM
#include <fmt/std.h>
#else
#include <fmt/core.h>
namespace fmt {
template <>
struct formatter<ghc::filesystem::path> : formatter<std::string>
{
template <typename FormatContext>
auto format(const ghc::filesystem::path& val, FormatContext& ctx) const {
return formatter<std::string>::format(val.u8string(), ctx);
}
};
}
#endif

1
src/common/fs.h
View File

@ -20,6 +20,7 @@ namespace fs {
#else
#include <ghc/filesystem.hpp>
namespace fs = ghc::filesystem;
#endif

4
src/common/i18n.cpp
View File

@ -37,9 +37,9 @@
#include <utility>
#include <algorithm>
#include "file.h"
#include <logging/oxen_logger.h>
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "i18n"
static auto logcat = oxen::log::Cat("i18n");
#define MAX_LANGUAGE_SIZE 16

4
src/common/json_util.h
View File

@ -41,13 +41,13 @@
} \
else \
{ \
LOG_ERROR("Field " << #name << " found in JSON, but not " << #jtype); \
oxen::log::error(logcat, "Field {} found in JSON, but not {}", #name, #jtype); \
return false; \
} \
} \
else if (mandatory) \
{ \
LOG_ERROR("Field " << #name << " not found in JSON"); \
oxen::log::error(logcat, "Field {} not found in JSON", #name); \
return false; \
} while(0)

2
src/common/meta.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <string>
#include <oxenmq/variant.h>
#include <oxenc/variant.h>
#include <array>
#include <typeinfo>
#ifdef __GNUG__

6
src/common/notify.cpp
View File

@ -28,15 +28,15 @@
#include "string_util.h"
#include "epee/misc_log_ex.h"
#include "logging/oxen_logger.h"
#include "spawn.h"
#include "notify.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "notify"
namespace tools
{
static auto logcat = log::Cat("notify");
/*
TODO:
- Improve tokenization to handle paths containing whitespaces, quotes, etc.

3
src/common/notify.h
View File

@ -32,6 +32,7 @@
#include <string_view>
#include <vector>
#include <fmt/core.h>
#include "common/format.h"
#include "fs.h"
namespace tools
@ -57,7 +58,7 @@ private:
template <typename T, typename... MoreTags>
static void replace_tags(std::vector<std::string>& margs, std::string_view tag, const T& value, MoreTags&&... more) {
replace_tag(margs, tag, fmt::format("{}", value));
replace_tag(margs, tag, "{}"_format(value));
if constexpr (sizeof...(MoreTags) > 0)
replace_tags(margs, std::forward<MoreTags>(more)...);
}

141
src/common/perf_timer.cpp
View File

@ -1,141 +0,0 @@
// Copyright (c) 2016-2019, The Monero Project
// Copyright (c) 2018, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <chrono>
#include <vector>
#include "perf_timer.h"
using namespace std::literals;
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "perf"
#define PERF_LOG_ALWAYS(level, cat, x) \
el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, el::base::DispatchAction::FileOnlyLog).construct(cat) << x
namespace tools
{
el::Level performance_timer_log_level = el::Level::Info;
static thread_local std::vector<LoggingPerformanceTimer*>* performance_timers = nullptr;
void set_performance_timer_log_level(el::Level level)
{
if (level != el::Level::Debug && level != el::Level::Trace && level != el::Level::Info
&& level != el::Level::Warning && level != el::Level::Error && level != el::Level::Fatal)
{
MERROR("Wrong log level: " << el::LevelHelper::convertToString(level) << ", using Info");
level = el::Level::Info;
}
performance_timer_log_level = level;
}
PerformanceTimer::PerformanceTimer(bool paused): started(true)
{
if (!paused)
since = std::chrono::steady_clock::now();
}
LoggingPerformanceTimer::LoggingPerformanceTimer(const std::string &s, const std::string &cat, uint64_t unit, el::Level l): PerformanceTimer(), name(s), cat(cat), unit(unit), level(l)
{
const bool log = ELPP->vRegistry()->allowed(level, cat.c_str());
if (!performance_timers)
{
if (log)
PERF_LOG_ALWAYS(level, cat.c_str(), "PERF ----------");
performance_timers = new std::vector<LoggingPerformanceTimer*>();
performance_timers->reserve(16); // how deep before realloc
}
else
{
LoggingPerformanceTimer* pt = performance_timers->back();
if (!pt->started && pt->since)
{
if (log)
{
size_t size = 0; for (const auto *tmp: *performance_timers) if (tmp->since) ++size;
PERF_LOG_ALWAYS(pt->level, cat.c_str(), "PERF " << std::string((size-1) * 2, ' ') << " " << pt->name);
}
pt->started = true;
}
}
performance_timers->push_back(this);
}
LoggingPerformanceTimer::~LoggingPerformanceTimer()
{
pause();
performance_timers->pop_back();
const bool log = ELPP->vRegistry()->allowed(level, cat.c_str());
if (log)
{
char s[12];
snprintf(s, sizeof(s), "%8llu ", static_cast<unsigned long long>(elapsed.count() / (1000000000 / unit)));
size_t size = 0; for (const auto *tmp: *performance_timers) if (tmp->since || tmp==this) ++size;
PERF_LOG_ALWAYS(level, cat.c_str(), "PERF " << s << std::string(size * 2, ' ') << " " << name);
}
if (performance_timers->empty())
{
delete performance_timers;
performance_timers = nullptr;
}
}
void PerformanceTimer::pause()
{
if (!since)
return;
auto now = std::chrono::steady_clock::now();
elapsed += now - *since;
since.reset();
}
void PerformanceTimer::resume()
{
if (!since)
since = std::chrono::steady_clock::now();
}
void PerformanceTimer::reset()
{
elapsed = 0ns;
if (since)
since = std::chrono::steady_clock::now();
}
std::chrono::nanoseconds PerformanceTimer::value() const
{
auto ret = elapsed;
if (since)
ret += std::chrono::steady_clock::now() - *since;
return ret;
}
} // namespace tools

84
src/common/perf_timer.h
View File

@ -1,84 +0,0 @@
// Copyright (c) 2016-2019, The Monero Project
// Copyright (c) 2018, The Loki Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <optional>
#include <string>
#include <cstdio>
#include <cstdint>
#include "epee/misc_log_ex.h"
namespace tools
{
extern el::Level performance_timer_log_level;
class PerformanceTimer
{
public:
PerformanceTimer(bool paused = false);
void pause();
void resume();
void reset();
std::chrono::nanoseconds value() const;
std::chrono::duration<double> seconds() const { return value(); }
protected:
std::optional<std::chrono::steady_clock::time_point> since;
std::chrono::nanoseconds elapsed;
bool started;
};
class LoggingPerformanceTimer: public PerformanceTimer
{
public:
LoggingPerformanceTimer(const std::string &s, const std::string &cat, uint64_t unit, el::Level l = el::Level::Info);
~LoggingPerformanceTimer();
private:
std::string name;
std::string cat;
uint64_t unit;
el::Level level;
};
void set_performance_timer_log_level(el::Level level);
#define PERF_TIMER_UNIT(name, unit) tools::LoggingPerformanceTimer pt_##name(#name, "perf." OXEN_DEFAULT_LOG_CATEGORY, unit, tools::performance_timer_log_level)
#define PERF_TIMER_UNIT_L(name, unit, l) tools::LoggingPerformanceTimer pt_##name(#name, "perf." OXEN_DEFAULT_LOG_CATEGORY, unit, l)
#define PERF_TIMER(name) PERF_TIMER_UNIT(name, 1000000)
#define PERF_TIMER_L(name, l) PERF_TIMER_UNIT_L(name, 1000000, l)
#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr<tools::LoggingPerformanceTimer> pt_##name(new tools::LoggingPerformanceTimer(#name, "perf." OXEN_DEFAULT_LOG_CATEGORY, unit, el::Level::Info))
#define PERF_TIMER_START(name) PERF_TIMER_START_UNIT(name, 1000000)
#define PERF_TIMER_STOP(name) do { pt_##name.reset(NULL); } while(0)
#define PERF_TIMER_PAUSE(name) pt_##name->pause()
#define PERF_TIMER_RESUME(name) pt_##name->resume()
}

35
src/common/scoped_message_writer.cpp
View File

@ -1,23 +1,30 @@
#include "scoped_message_writer.h"
#include "common/format.h"
tools::scoped_message_writer::~scoped_message_writer()
namespace tools {
static auto logcat = log::Cat("msgwriter");
scoped_message_writer& scoped_message_writer::flush()
{
if (m_flush)
if (!m_content.empty())
{
m_flush = false;
logcat->log(m_log_level, "{}{}", m_prefix, m_content);
MCLOG_FILE(m_log_level, "msgwriter", m_oss.str());
if (epee::console_color_default == m_color)
{
std::cout << m_oss.str();
if (m_color) {
rdln::suspend_readline pause_readline;
fmt::print(fg(*m_color), "{}{}\n", m_prefix, m_content);
}
else
{
rdln::suspend_readline pause_readline;
set_console_color(m_color, m_bright);
std::cout << m_oss.str();
epee::reset_console_color();
}
std::cout << std::endl;
fmt::print("{}{}\n", m_prefix, m_content);
m_content.clear();
}
return *this;
}
scoped_message_writer::~scoped_message_writer()
{
flush();
}
}

132
src/common/scoped_message_writer.h
View File

@ -28,9 +28,11 @@
#pragma once
#include "epee/misc_log_ex.h"
#include "epee/readline_suspend.h"
#include "epee/misc_log_ex.h"
#include <iostream>
#include "logging/oxen_logger.h"
#include <fmt/color.h>
namespace tools
{
@ -41,62 +43,116 @@ namespace tools
class scoped_message_writer
{
private:
bool m_flush;
std::ostringstream m_oss;
epee::console_colors m_color;
bool m_bright;
el::Level m_log_level;
std::string m_prefix;
std::string m_content;
std::optional<fmt::terminal_color> m_color;
oxen::log::Level m_log_level;
public:
scoped_message_writer(
epee::console_colors color = epee::console_color_default
, bool bright = false
, std::string prefix = {}
, el::Level log_level = el::Level::Info
)
: m_flush(true)
, m_color(color)
, m_bright(bright)
, m_log_level(log_level)
explicit scoped_message_writer(
std::optional<fmt::terminal_color> color = std::nullopt,
std::string prefix = "",
log::Level log_level = log::Level::info
)
: m_color{color}, m_log_level{log_level}, m_prefix{std::move(prefix)}
{}
scoped_message_writer(scoped_message_writer&& o)
: m_prefix{std::move(o.m_prefix)},
m_content{std::move(o.m_content)},
m_color{o.m_color},
m_log_level{o.m_log_level}
{
m_oss << prefix;
o.m_content.clear();
}
scoped_message_writer(scoped_message_writer&& rhs)
: m_flush(std::move(rhs.m_flush))
, m_oss(std::move(rhs.m_oss))
, m_color(std::move(rhs.m_color))
, m_log_level(std::move(rhs.m_log_level))
{
rhs.m_flush = false;
}
scoped_message_writer(scoped_message_writer& rhs) = delete;
scoped_message_writer& operator=(scoped_message_writer& rhs) = delete;
scoped_message_writer(const scoped_message_writer& rhs) = delete;
scoped_message_writer& operator=(const scoped_message_writer& rhs) = delete;
scoped_message_writer& operator=(scoped_message_writer&& rhs) = delete;
template<typename T>
std::ostream& operator<<(const T& val)
/// Appends a message and returns *this (so that it can be chained). If called with more than 1
/// argument then the first argument is fmt::format'ed with the remaining arguments.
template <typename... T>
scoped_message_writer& append(std::string_view msg, T&&... args)
{
m_oss << val;
return m_oss;
if constexpr (sizeof...(T))
fmt::format_to(std::back_inserter(m_content), msg, std::forward<T>(args)...);
else
m_content.append(msg);
return *this;
}
/// Same as .append(msg). (Doesn't format, just like the single-argument .append(msg)).
scoped_message_writer& operator+=(std::string_view msg) { return append(msg); }
/// Essentially the same as +=, but can only be used on an rvalue instance of the object, so that
/// you can do things like: `scoped_message_writer{} + "abc"`, which feels more natural than
/// `scoped_message_writer{} += "abc"`.
scoped_message_writer&& operator+(std::string_view msg) && { append(msg); return std::move(*this); }
/// Flushes the current message to output and resets it. This is normally not called explicitly
/// but rather implicitly when the object is destroyed.
scoped_message_writer& flush();
/// Prints the complete message on destruction.
~scoped_message_writer();
};
inline scoped_message_writer success_msg_writer(bool color = true)
template <typename... T>
scoped_message_writer msg_writer(std::optional<fmt::terminal_color> color = std::nullopt, T&&... args)
{
return scoped_message_writer(color ? epee::console_color_green : epee::console_color_default, false, std::string(), el::Level::Info);
scoped_message_writer writer{color};
if constexpr (sizeof...(T))
writer.append(std::forward<T>(args)...);
return writer;
}
inline scoped_message_writer msg_writer(epee::console_colors color = epee::console_color_default)
template <typename... T>
scoped_message_writer msg_writer(std::string_view msg, T&&... args)
{
return scoped_message_writer(color, false, std::string(), el::Level::Info);
return msg_writer(std::nullopt, msg, std::forward<T>(args)...);
}
inline scoped_message_writer fail_msg_writer()
constexpr std::optional<fmt::terminal_color> success_color{fmt::terminal_color::green};
constexpr std::optional<fmt::terminal_color> fail_color{fmt::terminal_color::red};
/// Constructs and returns a scoped_message_writer for a typical success message, with or without
/// color, as specified by the first argument. If additional arguments are provided they will be
/// passed to append(...) to set a message (or formatted message, if multiple arguments are given).
///
/// (We deduce the Bool argument here to avoid implicit conversion to bool from non-bool values).
template <typename Bool, typename... T, std::enable_if_t<std::is_same_v<Bool, bool>, int> = 0>
scoped_message_writer success_msg_writer(Bool color, T&&... args)
{
return scoped_message_writer(epee::console_color_red, true, "Error: ", el::Level::Error);
auto writer = msg_writer(color ? success_color : std::nullopt);
if constexpr (sizeof...(T))
writer.append(std::forward<T>(args)...);
return writer;
}
inline scoped_message_writer success_msg_writer()
{
return success_msg_writer(true);
}
/// Same as above, but for calling without just a message (with a bool). Color will be true.
template <typename... T>
scoped_message_writer success_msg_writer(std::string_view msg, T&&... args)
{
return success_msg_writer(true, msg, std::forward<T>(args)...);
}
/// Constructs and returns a scoped_message_writer for a typical error message. Color will be
/// enabled and the message will be prefixed with "Error: ". Given arguments, if any, are passed to
/// .append() and so can specify either a single unformatted string, or a format string + format
/// arguments.
template <typename... T>
scoped_message_writer fail_msg_writer(T&&... args)
{
scoped_message_writer writer{fail_color, "Error: ", spdlog::level::err};
if constexpr (sizeof...(T))
writer.append(std::forward<T>(args)...);
return writer;
}
} // namespace tools

4
src/common/sha256sum.cpp
View File

@ -10,7 +10,7 @@ namespace tools {
bool sha256sum_str(std::string_view data, crypto::hash &hash)
{
crypto_hash_sha256(
reinterpret_cast<unsigned char*>(hash.data),
hash.data(),
reinterpret_cast<const unsigned char*>(data.data()),
data.size());
return true;
@ -42,7 +42,7 @@ namespace tools {
size_left -= read_size;
}
f.close();
crypto_hash_sha256_final(&st, reinterpret_cast<unsigned char*>(hash.data));
crypto_hash_sha256_final(&st, hash.data());
return true;
}

9
src/common/signal_handler.h
View File

@ -3,6 +3,11 @@
#include <csignal>
#include <functional>
#include <mutex>
#include "logging/oxen_logger.h"
#ifdef _WIN32
#include "windows.h"
#endif
namespace tools {
@ -15,7 +20,7 @@ namespace tools {
template<typename T>
static bool install(T t)
{
#if defined(WIN32)
#ifdef _WIN32
bool r = TRUE == ::SetConsoleCtrlHandler(&win_handler, TRUE);
if (r)
{
@ -47,7 +52,7 @@ namespace tools {
}
else
{
MGINFO_RED("Got control signal " << type << ". Exiting without saving...");
log::info(globallogcat, fg(fmt::terminal_color::red), "Got control signal {}. Exiting without saving...", type);
return FALSE;
}
return TRUE;

26
src/common/spawn.cpp
View File

@ -37,17 +37,17 @@
#endif
#include "epee/misc_log_ex.h"
#include "logging/oxen_logger.h"
#include "util.h"
#include "spawn.h"
#include "oxen.h"
#include "string_util.h"
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "spawn"
namespace tools
{
static auto logcat = log::Cat("spawn");
#ifndef _WIN32
static void closefrom(int fd)
{
@ -83,7 +83,7 @@ int spawn(const fs::path& filename, const std::vector<std::string>& args, bool w
// wchar_t* but out input is utf-8). Shame on you for this garbage API, Windows.
if (!CreateProcessA(filename.string().c_str(), commandLine, nullptr, nullptr, false, 0, nullptr, nullptr, &si, &pi))
{
MERROR("CreateProcess failed. Error code " << GetLastError());
log::error(logcat, "CreateProcess failed. Error code {}", GetLastError());
return -1;
}
@ -100,18 +100,18 @@ int spawn(const fs::path& filename, const std::vector<std::string>& args, bool w
DWORD result = WaitForSingleObject(pi.hProcess, INFINITE);
if (result != WAIT_OBJECT_0)
{
MERROR("WaitForSingleObject failed. Result " << result << ", error code " << GetLastError());
log::error(logcat, "WaitForSingleObject failed. Result {}, error code {}", result, GetLastError());
return -1;
}
DWORD exitCode;
if (!GetExitCodeProcess(pi.hProcess, &exitCode))
{
MERROR("GetExitCodeProcess failed. Error code " << GetLastError());
log::error(logcat, "GetExitCodeProcess failed. Error code {}", GetLastError());
return -1;
}
MINFO("Child exited with " << exitCode);
log::info(logcat, "Child exited with {}", exitCode);
return static_cast<int>(exitCode);
#else
std::vector<char*> argv(args.size() + 1);
@ -122,7 +122,7 @@ int spawn(const fs::path& filename, const std::vector<std::string>& args, bool w
pid_t pid = fork();
if (pid < 0)
{
MERROR("Error forking: " << strerror(errno));
log::error(logcat, "Error forking: {}", strerror(errno));
return -1;
}
@ -133,7 +133,7 @@ int spawn(const fs::path& filename, const std::vector<std::string>& args, bool w
close(0);
char *envp[] = {NULL};
execve(filename.c_str(), argv.data(), envp);
MERROR("Failed to execve: " << strerror(errno));
log::error(logcat, "Failed to execve: {}", strerror(errno));
return -1;
}
@ -151,22 +151,22 @@ int spawn(const fs::path& filename, const std::vector<std::string>& args, bool w
int wstatus = 0;
pid_t w = waitpid(pid, &wstatus, WUNTRACED | WCONTINUED);
if (w < 0) {
MERROR("Error waiting for child: " << strerror(errno));
log::error(logcat, "Error waiting for child: {}", strerror(errno));
return -1;
}
if (WIFEXITED(wstatus))
{
MINFO("Child exited with " << WEXITSTATUS(wstatus));
log::info(logcat, "Child exited with {}", WEXITSTATUS(wstatus));
return WEXITSTATUS(wstatus);
}
if (WIFSIGNALED(wstatus))
{
MINFO("Child killed by " << WEXITSTATUS(wstatus));
log::info(logcat, "Child killed by {}", WEXITSTATUS(wstatus));
return WEXITSTATUS(wstatus);
}
}
}
MERROR("Secret passage found");
log::error(logcat, "Secret passage found");
return -1;
#endif
}

52
src/common/string_util.cpp
View File

@ -1,7 +1,6 @@
#include "string_util.h"
#include <fmt/core.h>
#include <cassert>
#include <iomanip>
#include <sstream>
namespace tools {
@ -80,66 +79,39 @@ std::string lowercase_ascii_string(std::string_view src)
}
std::string friendly_duration(std::chrono::nanoseconds dur) {
std::ostringstream os;
std::string friendly;
auto append = std::back_inserter(friendly);
bool some = false;
if (dur >= 24h) {
os << dur / 24h << 'd';
fmt::format_to(append, "{}d", dur / 24h);
dur %= 24h;
some = true;
}
if (dur >= 1h || some) {
os << dur / 1h << 'h';
fmt::format_to(append, "{}h", dur / 1h);
dur %= 1h;
some = true;
}
if (dur >= 1min || some) {
os << dur / 1min << 'm';
fmt::format_to(append, "{}m", dur / 1min);
dur %= 1min;
some = true;
}
if (some || dur == 0s) {
// If we have >= minutes or its exactly 0 seconds then don't bother with fractional seconds
os << dur / 1s << 's';
fmt::format_to(append, "{}s", dur / 1s);
} else {
double seconds = std::chrono::duration<double>(dur).count();
os.precision(3);
if (dur >= 1s)
os << seconds << "s";
fmt::format_to(append, "{:.3f}s", seconds);
else if (dur >= 1ms)
os << seconds * 1000 << "ms";
fmt::format_to(append, "{:.3f}ms", seconds * 1000);
else if (dur >= 1us)
os << seconds * 1'000'000 << u8"µs";
fmt::format_to(append, "{:.3f}µs", seconds * 1'000'000);
else
os << seconds * 1'000'000'000 << "ns";
fmt::format_to(append, "{:.0f}ns", seconds * 1'000'000'000);
}
return os.str();
return friendly;
}
std::string short_duration(std::chrono::duration<double> dur) {
std::ostringstream os;
os << std::fixed << std::setprecision(1);
if (dur >= 36h)
os << dur / 24h;
else if (dur >= 90min)
os << dur / 1h;
else if (dur >= 90s)
os << dur / 1min;
else if (dur >= 1s)
os << dur / 1s;
else if (dur >= 100ms)
os << std::setprecision(0) << dur / 1ms;
else if (dur >= 1ms)
os << dur / 1ms;
else if (dur >= 100us)
os << std::setprecision(0) << dur / 1us;
else if (dur >= 1us)
os << dur / 1us;
else if (dur >= 1ns)
os << std::setprecision(0) << dur / 1ns;
else
os << "0s";
return os.str();
}
}

54
src/common/string_util.h
View File

@ -4,9 +4,9 @@
#include <cstring>
#include <iterator>
#include <charconv>
#include <sstream>
#include <chrono>
#include <cassert>
#include <fmt/format.h>
#include "epee/span.h" // epee
namespace tools {
@ -60,30 +60,28 @@ std::vector<std::string_view> split(std::string_view str, std::string_view delim
std::vector<std::string_view> split_any(std::string_view str, std::string_view delims, bool trim = false);
/// Joins [begin, end) with a delimiter and returns the resulting string. Elements can be anything
/// that can be sent to an ostream via `<<`.
/// that can be formatted. Semi-deprecated: this just uses fmt to join.
template <typename It>
std::string join(std::string_view delimiter, It begin, It end) {
std::ostringstream o;
if (begin != end)
o << *begin++;
while (begin != end)
o << delimiter << *begin++;
return o.str();
return fmt::format("{}", fmt::join(begin, end, delimiter));
}
/// Wrapper around the above that takes a container and passes c.begin(), c.end() to the above.
/// Same as the above, but works on a container. Just use fmt::join.
template <typename Container>
std::string join(std::string_view delimiter, const Container& c) { return join(delimiter, c.begin(), c.end()); }
std::string join(std::string_view delimiter, const Container& c) {
return fmt::format("{}", fmt::join(c, delimiter));
}
/// Similar to join(), but first applies a transformation to each element.
template <typename It, typename UnaryOperation>
std::string join_transform(std::string_view delimiter, It begin, It end, UnaryOperation transform) {
std::ostringstream o;
std::string result;
auto append = std::back_inserter(result);
if (begin != end)
o << transform(*begin++);
result = fmt::format("{}", transform(*begin++));
while (begin != end)
o << delimiter << transform(*begin++);
return o.str();
fmt::format_to(append, "{}{}", delimiter, transform(*begin++));
return result;
}
/// Wrapper around the above that takes a container and passes c.begin(), c.end().
@ -92,22 +90,23 @@ std::string join_transform(std::string_view delimiter, const Container& c, Unary
return join_transform(delimiter, c.begin(), c.end(), std::forward<UnaryOperation>(transform));
}
/// Concatenates a bunch of random values together with delim as a separator via << operator.
/// Concatenates a bunch of random values together with delim as a separator via fmt::format.
/// Returns the result as a string.
template <typename T, typename... Ts>
std::string join_stuff(std::string_view delim, T&& first, Ts&&... stuff) {
std::ostringstream o;
o << std::forward<T>(first);
((o << delim << std::forward<Ts>(stuff)), ...);
return o.str();
std::string result = fmt::format(std::forward<T>(first));
auto append = std::back_inserter(result);
(fmt::format_to(append, "{}{}", delim, std::forward<Ts>(stuff)), ...);
return result;
}
/// Concatenates arguments via << operator, returns as a string.
/// Concatenates arguments via fmt::format operator, returns as a string.
template <typename... T>
std::string concat(T&&... stuff) {
std::ostringstream o;
(o << ... << std::forward<T>(stuff));
return o.str();
std::string result;
auto append = std::back_inserter(result);
(fmt::format_to(append, "{}", std::forward<T>(stuff)), ...);
return result;
}
/// Simple version of whitespace trimming: mutates the given string view to remove leading
@ -128,15 +127,6 @@ bool parse_int(const std::string_view str, T& value, int base = 10) {
return true;
}
/// Converts an integer value into a string via std::to_chars (i.e. without locale).
template <typename T, typename = std::enable_if_t<std::is_integral_v<T>>>
std::string int_to_string(const T& value, int base = 10) {
char buf[8*sizeof(T) + std::is_signed_v<T>]; // maximum possible size with smallest possible base (2)
auto [p, ec] = std::to_chars(std::begin(buf), std::end(buf), value, base);
assert(ec == std::errc{}); // Our buffer should be big enough for anything
return {buf, p};
}
/// Returns a string_view that views the data of the given object; this is not something you want to
/// do unless the struct is specifically design to be used this way. The value must be a standard
/// layout type; it should really require is_trivial, too, but we have classes (like crypto keys)

4
src/common/threadpool.cpp
View File

@ -26,6 +26,7 @@
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "epee/misc_log_ex.h"
#include "logging/oxen_logger.h"
#include "common/threadpool.h"
#include "cryptonote_config.h"
@ -36,6 +37,7 @@ static thread_local bool is_leaf = false;
namespace tools
{
threadpool::threadpool(unsigned int max_threads) : running(true), active(0) {
create(max_threads);
}
@ -112,7 +114,7 @@ threadpool::waiter::~waiter()
{
std::unique_lock lock{mt};
if (num)
MERROR("wait should have been called before waiter dtor - waiting now");
log::error(globallogcat, "wait should have been called before waiter dtor - waiting now");
}
catch (...) { /* ignore */ }
try

16
src/common/util.cpp
View File

@ -33,6 +33,7 @@
#include <string>
#include <iomanip>
#include <thread>
#include <fmt/color.h>
#include <date/date.h>
@ -42,6 +43,7 @@
#include "util.h"
#include "epee/readline_buffer.h"
#include "epee/misc_log_ex.h"
#include "logging/oxen_logger.h"
#include "string_util.h"
#include "i18n.h"
@ -51,11 +53,9 @@
#include <gnu/libc-version.h>
#endif
#undef OXEN_DEFAULT_LOG_CATEGORY
#define OXEN_DEFAULT_LOG_CATEGORY "util"
namespace tools
{
static auto logcat = log::Cat("util");
bool disable_core_dumps()
{
@ -65,7 +65,7 @@ namespace tools
rlimit.rlim_cur = rlimit.rlim_max = 0;
if (setrlimit(RLIMIT_CORE, &rlimit))
{
MWARNING("Failed to disable core dumps");
log::warning(logcat, "Failed to disable core dumps");
return false;
}
#endif
@ -78,7 +78,7 @@ namespace tools
struct rlimit rlim;
if (getrlimit(RLIMIT_MEMLOCK, &rlim) < 0)
{
MERROR("Failed to determine the lockable memory limit");
log::error(logcat, "Failed to determine the lockable memory limit");
return -1;
}
return rlim.rlim_cur;
@ -89,12 +89,10 @@ namespace tools
bool on_startup()
{
mlog_configure("", true);
#ifdef __GLIBC__
const char *ver = ::gnu_get_libc_version();
if (!strcmp(ver, "2.25"))
MCLOG_RED(el::Level::Warning, "global", "Running with glibc " << ver << ", hangs may occur - change glibc version if possible");
log::warning(logcat, fg(fmt::terminal_color::red), "Running with glibc {}, hangs may occur - change glibc version if possible", ver);
#endif
return true;
@ -164,7 +162,7 @@ namespace tools
}
else
{
return {};
return std::nullopt;
}
}

26
src/common/util.h
View File

@ -90,8 +90,23 @@ namespace tools
dest += sizeof(T);
}
struct crypto_bytes_base_helper {
template <size_t S>
static std::true_type check(crypto::bytes<S, true>*);
static std::false_type check(...);
};
template <typename T>
constexpr bool is_crypto_bytes_derived = decltype(crypto_bytes_base_helper::check((T*) nullptr))::value;
// Copy the data out of a crypto::bytes<N, true>-derived type.
template <typename T, std::enable_if_t<is_crypto_bytes_derived<T>, int> = 0>
void memcpy_one(char*& dest, const T& t) {
std::memcpy(dest, t.data(), t.size());
dest += t.size();
}
// Copy a class byte-for-byte (but only if it is standard layout and has byte alignment)
template <typename T, std::enable_if_t<std::is_class<T>::value, int> = 0>
template <typename T, std::enable_if_t<std::is_class<T>::value && !is_crypto_bytes_derived<T>, int> = 0>
void memcpy_one(char*& dest, const T& t) {
// We don't *actually* require byte alignment here but it's quite possibly an error (i.e.
// passing in a type containing integer members) so disallow it.
@ -106,6 +121,13 @@ namespace tools
for (const T &t : arr)
memcpy_one(dest, t);
}
template <typename T, typename = void>
constexpr size_t memcpy_size = sizeof(T);
template <typename T>
inline constexpr size_t memcpy_size<T, std::enable_if_t<is_crypto_bytes_derived<T>>>
= T::size();
}
// Does a memcpy of one or more values into a char array; for any given values that are basic
@ -118,7 +140,7 @@ namespace tools
// you have a contained type with a larger alignment, which is probably an integer.
template <typename... T>
auto memcpy_le(const T &...t) {
std::array<char, (0 + ... + sizeof(T))> r;
std::array<char, (0 + ... + detail::memcpy_size<T>)> r;
char* dest = r.data();
(..., detail::memcpy_one(dest, t));
return r;

3
src/crypto/CMakeLists.txt
View File

@ -31,6 +31,7 @@ add_library(cncrypto
aesb.c
blake256.c
chacha.c
chacha.cpp
crypto-ops-data.c
crypto-ops.c
crypto.cpp
@ -40,6 +41,7 @@ add_library(cncrypto
hash-extra-jh.c
hash-extra-skein.c
hash.c
hash.cpp
hmac-keccak.c
jh.c
keccak.c
@ -66,6 +68,7 @@ target_link_libraries(cncrypto
Boost::thread
sodium
PRIVATE
logging
extra)
if (CMAKE_CXX_COMPILER_ID MATCHES Clang OR CMAKE_CXX_COMPILER_ID STREQUAL GNU)

82
src/crypto/base.h Normal file
View File

@ -0,0 +1,82 @@
#pragma once
#include "common/format.h"
#include "common/formattable.h"
#include "common/hex.h"
#include <array>
#include <type_traits>
namespace crypto {
/// constexpr null (all-0) value for various crypto types; use as `crypto::null<crypto::whatever>`.
template <typename T, typename = std::enable_if_t<std::is_standard_layout_v<T> && std::is_default_constructible_v<T>>>
constexpr T null{};
// Base type for fixed-byte quantities (points, scalars, signatures, hashes). The bool controls
// whether the type should have ==, !=, std::hash, and to_hex_string.
template <size_t Bytes, bool MemcmpHashHex = false>
struct alignas(size_t) bytes {
std::array<unsigned char, Bytes> data_;
unsigned char* data() { return data_.data(); }
const unsigned char* data() const { return data_.data(); }
static constexpr size_t size() { return Bytes; }
auto begin() { return data_.begin(); }
auto begin() const { return data_.begin(); }
auto cbegin() const { return data_.cbegin(); }
auto end() { return data_.end(); }
auto end() const { return data_.end(); }
auto cend() const { return data_.cend(); }
// Set the bytes to all 0's
void zero() { data_.fill(0); }
unsigned char& operator[](size_t i) { return data_[i]; }
const unsigned char& operator[](size_t i) const { return data_[i]; }
static constexpr bool compare_hash_hex = MemcmpHashHex;
};
template <typename T, typename = void>
constexpr bool has_compare_hash_hex = false;
template <typename T>
inline constexpr bool has_compare_hash_hex<T, std::enable_if_t<T::compare_hash_hex>> = true;
template <typename Left, typename Right, typename = void>
constexpr bool are_comparable_v = false;
template <typename L, typename R>
inline constexpr bool are_comparable_v<L, R, std::enable_if_t<std::is_same_v<L, R> && has_compare_hash_hex<L>>> = true;
template <typename L, typename R, std::enable_if_t<are_comparable_v<L, R>, int> = 0>
bool operator==(const L& left, const R& right) {
return left.data_ == right.data_;
}
template <typename L, typename R, std::enable_if_t<are_comparable_v<L, R>, int> = 0>
bool operator!=(const L& left, const R& right) {
return left.data_ != right.data_;
}
template <typename L, typename R, std::enable_if_t<are_comparable_v<L, R>, int> = 0>
bool operator<(const L& left, const R& right) {
return left.data_ < right.data_;
}
template <typename T, typename = std::enable_if_t<has_compare_hash_hex<T>>>
std::string to_hex_string(const T& val) { return "<{}>"_format(tools::type_to_hex(val)); }
template <typename T>
struct raw_hasher {
static_assert(T::compare_hash_hex);
static_assert(std::is_standard_layout_v<T>);
static_assert(sizeof(T) >= sizeof(size_t));
static_assert(alignof(T) >= sizeof(size_t));
size_t operator()(const T& val) const {
return *reinterpret_cast<const size_t*>(val.data());
}
};
}
template <typename T>
inline constexpr bool formattable::via_to_hex_string<T, std::enable_if_t<crypto::has_compare_hash_hex<T>>>
= true;

43
src/crypto/chacha.cpp Normal file
View File

@ -0,0 +1,43 @@
#include "chacha.h"
#include "hash.h"
#include "cn_heavy_hash.hpp"
namespace crypto {
static_assert(sizeof(chacha_key) == CHACHA_KEY_SIZE && sizeof(chacha_iv) == CHACHA_IV_SIZE, "Invalid structure size");
void chacha8(const void* data, std::size_t length, const chacha_key& key, const chacha_iv& iv, char* cipher) {
chacha8(data, length, key.data(), reinterpret_cast<const uint8_t*>(&iv), cipher);
}
void chacha20(const void* data, std::size_t length, const chacha_key& key, const chacha_iv& iv, char* cipher) {
chacha20(data, length, key.data(), reinterpret_cast<const uint8_t*>(&iv), cipher);
}
void generate_chacha_key(std::string password, chacha_key& key, uint64_t kdf_rounds) {
return generate_chacha_key(password.data(), password.size(), key, kdf_rounds);
}
void generate_chacha_key(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) {
static_assert(sizeof(chacha_key) <= hash::size(), "Size of hash must be at least that of chacha_key");
epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash;
static thread_local cn_heavy_hash_v1 ctx;
ctx.hash(data, size, pwd_hash.data());
for (uint64_t n = 1; n < kdf_rounds; ++n)
ctx.hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data());
memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key));
}
void generate_chacha_key_prehashed(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) {
static_assert(sizeof(chacha_key) <= hash::size(), "Size of hash must be at least that of chacha_key");
epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash;
static thread_local cn_heavy_hash_v1 ctx;
ctx.hash(data, size, pwd_hash.data(), true);
for (uint64_t n = 1; n < kdf_rounds; ++n)
ctx.hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data());
memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key));
}
}

57
src/crypto/chacha.h
View File

@ -36,21 +36,20 @@
#define CHACHA_KEY_SIZE 32
#define CHACHA_IV_SIZE 8
#if defined(__cplusplus)
#include <memory.h>
#ifdef __cplusplus
extern "C" {
#endif
void chacha8(const void* data, size_t length, const uint8_t* key, const uint8_t* iv, char* cipher);
void chacha20(const void* data, size_t length, const uint8_t* key, const uint8_t* iv, char* cipher);
#ifdef __cplusplus
}
#include "epee/memwipe.h"
#include "epee/mlocker.h"
#include "hash.h"
#include "cn_heavy_hash.hpp"
namespace crypto {
extern "C" {
#endif
void chacha8(const void* data, size_t length, const uint8_t* key, const uint8_t* iv, char* cipher);
void chacha20(const void* data, size_t length, const uint8_t* key, const uint8_t* iv, char* cipher);
#if defined(__cplusplus)
}
using chacha_key = epee::mlocked<tools::scrubbed_arr<uint8_t, CHACHA_KEY_SIZE>>;
@ -58,39 +57,15 @@ namespace crypto {
uint8_t data[CHACHA_IV_SIZE];
};
static_assert(sizeof(chacha_key) == CHACHA_KEY_SIZE && sizeof(chacha_iv) == CHACHA_IV_SIZE, "Invalid structure size");
using ::chacha8;
using ::chacha20;
inline void chacha8(const void* data, std::size_t length, const chacha_key& key, const chacha_iv& iv, char* cipher) {
chacha8(data, length, key.data(), reinterpret_cast<const uint8_t*>(&iv), cipher);
}
void chacha8(const void* data, std::size_t length, const chacha_key& key, const chacha_iv& iv, char* cipher);
void chacha20(const void* data, std::size_t length, const chacha_key& key, const chacha_iv& iv, char* cipher);
inline void chacha20(const void* data, std::size_t length, const chacha_key& key, const chacha_iv& iv, char* cipher) {
chacha20(data, length, key.data(), reinterpret_cast<const uint8_t*>(&iv), cipher);
}
inline void generate_chacha_key(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) {
static_assert(sizeof(chacha_key) <= sizeof(hash), "Size of hash must be at least that of chacha_key");
epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash;
static thread_local cn_heavy_hash_v1 ctx;
ctx.hash(data, size, pwd_hash.data());
for (uint64_t n = 1; n < kdf_rounds; ++n)
ctx.hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data());
memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key));
}
inline void generate_chacha_key_prehashed(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) {
static_assert(sizeof(chacha_key) <= sizeof(hash), "Size of hash must be at least that of chacha_key");
epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash;
static thread_local cn_heavy_hash_v1 ctx;
ctx.hash(data, size, pwd_hash.data(), true);
for (uint64_t n = 1; n < kdf_rounds; ++n)
ctx.hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data());
memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key));
}
inline void generate_chacha_key(std::string password, chacha_key& key, uint64_t kdf_rounds) {
return generate_chacha_key(password.data(), password.size(), key, kdf_rounds);
}
void generate_chacha_key(std::string password, chacha_key& key, uint64_t kdf_rounds);
void generate_chacha_key(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds);
void generate_chacha_key_prehashed(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds);
}
#endif

8
src/crypto/cn_monero_slow_hash.c
View File

@ -695,7 +695,7 @@ void monero_hash_free_state(void)
* @param length the length in bytes of the data
* @param hash a pointer to a buffer in which the final 256 bit hash will be stored
*/
void cn_monero_hash(const void *data, size_t length, char *hash, int variant, int prehashed)
void cn_monero_hash(const void *data, size_t length, unsigned char *hash, int variant, int prehashed)
{
RDATA_ALIGN16 uint8_t expandedKey[AES_EXPANDED_KEY_SIZE]; /* These buffers are aligned to use later with SSE functions */
@ -1061,7 +1061,7 @@ STATIC INLINE void aligned_free(void *ptr)
}
#endif /* FORCE_USE_HEAP */
void cn_monero_hash(const void *data, size_t length, char *hash, int variant, int prehashed)
void cn_monero_hash(const void *data, size_t length, unsigned char *hash, int variant, int prehashed)
{
RDATA_ALIGN16 uint8_t expandedKey[240];
@ -1276,7 +1276,7 @@ STATIC INLINE void xor_blocks(uint8_t* a, const uint8_t* b)
U64(a)[1] ^= U64(b)[1];
}
void cn_monero_hash(const void *data, size_t length, char *hash, int variant, int prehashed)
void cn_monero_hash(const void *data, size_t length, unsigned char *hash, int variant, int prehashed)
{
uint8_t text[INIT_SIZE_BYTE];
uint8_t a[AES_BLOCK_SIZE];
@ -1468,7 +1468,7 @@ union cn_monero_hash_state {
};
#pragma pack(pop)
void cn_monero_hash(const void *data, size_t length, char *hash, int variant, int prehashed) {
void cn_monero_hash(const void *data, size_t length, unsigned char *hash, int variant, int prehashed) {
#ifndef FORCE_USE_HEAP
uint8_t long_state[MEMORY];
#else

4
src/crypto/cn_turtle_hash-amd64.inl
View File

@ -408,7 +408,7 @@ void slow_hash_free_state(uint32_t page_size)
* @param length the length in bytes of the data
* @param hash a pointer to a buffer in which the final 256 bit hash will be stored
*/
void cn_turtle_hash(const void *data, size_t length, char *hash, int light, int variant, int prehashed, uint32_t scratchpad, uint32_t iterations)
void cn_turtle_hash(const void *data, size_t length, unsigned char *hash, int light, int variant, int prehashed, uint32_t scratchpad, uint32_t iterations)
{
uint32_t TOTALBLOCKS = (CN_TURTLE_PAGE_SIZE / AES_BLOCK_SIZE);
uint32_t init_rounds = (scratchpad / INIT_SIZE_BYTE);
@ -428,7 +428,7 @@ void cn_turtle_hash(const void *data, size_t length, char *hash, int light, int
size_t i, j;
uint64_t *p = NULL;
static void (*const extra_hashes[4])(const void *, size_t, char *) =
static void (*const extra_hashes[4])(const void *, size_t, unsigned char *) =
{
hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein
};

8
src/crypto/cn_turtle_hash-arm.inl
View File

@ -217,7 +217,7 @@ STATIC INLINE void aligned_free(void *ptr)
}
#endif /* FORCE_USE_HEAP */
void cn_turtle_hash(const void *data, size_t length, char *hash, int light, int variant, int prehashed, uint32_t scratchpad, uint32_t iterations)
void cn_turtle_hash(const void *data, size_t length, unsigned char *hash, int light, int variant, int prehashed, uint32_t scratchpad, uint32_t iterations)
{
uint32_t TOTALBLOCKS = (CN_TURTLE_PAGE_SIZE / AES_BLOCK_SIZE);
uint32_t init_rounds = (scratchpad / INIT_SIZE_BYTE);
@ -244,7 +244,7 @@ void cn_turtle_hash(const void *data, size_t length, char *hash, int light, int
size_t i, j;
uint64_t *p = NULL;
static void (*const extra_hashes[4])(const void *, size_t, char *) =
static void (*const extra_hashes[4])(const void *, size_t, unsigned char *) =
{
hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein
};
@ -438,7 +438,7 @@ STATIC INLINE void xor_blocks(uint8_t* a, const uint8_t* b)
U64(a)[1] ^= U64(b)[1];
}
void cn_turtle_hash(const void *data, size_t length, char *hash, int light, int variant, int prehashed, uint32_t scratchpad, uint32_t iterations)
void cn_turtle_hash(const void *data, size_t length, unsigned char *hash, int light, int variant, int prehashed, uint32_t scratchpad, uint32_t iterations)
{
fprintf(stderr, "%s:%d OMG", __FILE__, __LINE__);
uint32_t init_rounds = (scratchpad / INIT_SIZE_BYTE);
@ -458,7 +458,7 @@ void cn_turtle_hash(const void *data, size_t length, char *hash, int light, int
size_t i, j;
uint8_t *p = NULL;
static void (*const extra_hashes[4])(const void *, size_t, char *) =
static void (*const extra_hashes[4])(const void *, size_t, unsigned char *) =
{
hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein
};

4
src/crypto/cn_turtle_hash-portable.inl
View File

@ -1,6 +1,6 @@
// Portable implementation as a fallback
static void (*const extra_hashes[4])(const void *, size_t, char *) = {
static void (*const extra_hashes[4])(const void *, size_t, unsigned char *) = {
hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein
};
@ -72,7 +72,7 @@ union cn_turtle_hash_state {
};
#pragma pack(pop)
void cn_turtle_hash(const void *data, size_t length, char *hash, int light, int variant, int prehashed, uint32_t scratchpad, uint32_t iterations)
void cn_turtle_hash(const void *data, size_t length, unsigned char *hash, int light, int variant, int prehashed, uint32_t scratchpad, uint32_t iterations)
{
uint32_t init_rounds = (scratchpad / INIT_SIZE_BYTE);
uint32_t aes_rounds = (iterations / 2);

362
src/crypto/crypto.cpp
View File

@ -38,11 +38,17 @@
#include <memory>
#include <stdexcept>
#include <sodium/utils.h>
#include <sodium/crypto_sign_ed25519.h>
#include <sodium/crypto_verify_32.h>
#include "common/varint.h"
#include "epee/warnings.h"
#include "crypto.h"
extern "C" {
#include "keccak.h"
#include "crypto-ops.h"
#include "random.h"
}
#include "hash.h"
@ -60,43 +66,26 @@ namespace {
namespace crypto {
using std::abort;
using std::int32_t;
using std::int64_t;
using std::size_t;
using std::uint32_t;
using std::uint64_t;
static_assert(sizeof(bytes<32>) == 32 && std::has_unique_object_representations_v<bytes<32>>);
static_assert(sizeof(bytes<64>) == 64 && std::has_unique_object_representations_v<bytes<64>>);
static_assert(sizeof(ec_point) == 32 && std::has_unique_object_representations_v<ec_point>);
static_assert(sizeof(ec_scalar) == 32 && std::has_unique_object_representations_v<ec_scalar>);
static_assert(sizeof(public_key) == 32 && std::has_unique_object_representations_v<public_key>);
static_assert(sizeof(secret_key_) == 32 && std::has_unique_object_representations_v<secret_key_>);
static_assert(sizeof(secret_key) == sizeof(secret_key_));
static_assert(sizeof(key_derivation) == 32 && std::has_unique_object_representations_v<key_derivation>);
static_assert(sizeof(key_image) == 32 && std::has_unique_object_representations_v<key_image>);
static_assert(sizeof(signature) == 64 && std::has_unique_object_representations_v<signature>);
static_assert(sizeof(ed25519_public_key) == crypto_sign_ed25519_PUBLICKEYBYTES && std::has_unique_object_representations_v<ed25519_public_key>);
static_assert(sizeof(ed25519_secret_key_) == crypto_sign_ed25519_SECRETKEYBYTES && std::has_unique_object_representations_v<ed25519_secret_key_>);
static_assert(sizeof(ed25519_signature) == 64 && std::has_unique_object_representations_v<ed25519_signature>);
extern "C" {
#include "crypto-ops.h"
#include "random.h"
bool ec_scalar::operator==(const ec_scalar& x) const {
return crypto_verify_32(data(), x.data()) == 0;
}
// These nasty dirty hacks are unspeakable disgusting. This is only here because all of these
// have a `.data` element, but it is a `char` instead of an `unsigned char`. So rather than
// change it to `unsigned char`, the author decided that he should overload `&` to do a
// reinterpret_cast. WTF.
//
// TODO: fix this garbage by making the ec_ types use unsigned char instead of char.
// EW!
static inline unsigned char *operator &(ec_point &point) {
return &reinterpret_cast<unsigned char &>(point);
}
// EW!
static inline const unsigned char *operator &(const ec_point &point) {
return &reinterpret_cast<const unsigned char &>(point);
}
// EW!
static inline unsigned char *operator &(ec_scalar &scalar) {
return &reinterpret_cast<unsigned char &>(scalar);
}
// EW!
static inline const unsigned char *operator &(const ec_scalar &scalar) {
return &reinterpret_cast<const unsigned char &>(scalar);
ec_scalar::operator bool() const {
return !sodium_is_zero(data(), size());
}
static std::mutex random_mutex;
@ -144,7 +133,7 @@ namespace crypto {
}
/* generate a random ]0..L[ scalar */
void random_scalar(ec_scalar &res) {
random_scalar(reinterpret_cast<unsigned char*>(res.data));
random_scalar(res.data());
}
ec_scalar random_scalar() {
@ -154,8 +143,14 @@ namespace crypto {
}
void hash_to_scalar(const void *data, size_t length, ec_scalar &res) {
cn_fast_hash(data, length, reinterpret_cast<hash &>(res));
sc_reduce32(&res);
cn_fast_hash(data, length, res.data());
sc_reduce32(res.data());
}
ec_scalar hash_to_scalar(const void *data, size_t length) {
ec_scalar x;
hash_to_scalar(data, length, x);
return x;
}
/*
@ -175,26 +170,26 @@ namespace crypto {
random_scalar(rng);
}
sec = rng;
sc_reduce32(&unwrap(sec)); // reduce in case second round of keys (sendkeys)
sc_reduce32(sec.data()); // reduce in case second round of keys (sendkeys)
ge_scalarmult_base(&point, &unwrap(sec));
ge_p3_tobytes(&pub, &point);
ge_scalarmult_base(&point, sec.data());
ge_p3_tobytes(pub.data(), &point);
return rng;
}
bool check_key(const public_key &key) {
ge_p3 point;
return ge_frombytes_vartime(&point, &key) == 0;
return ge_frombytes_vartime(&point, key.data()) == 0;
}
bool secret_key_to_public_key(const secret_key &sec, public_key &pub) {
ge_p3 point;
if (sc_check(&unwrap(sec)) != 0) {
if (sc_check(sec.data()) != 0) {
return false;
}
ge_scalarmult_base(&point, &unwrap(sec));
ge_p3_tobytes(&pub, &point);
ge_scalarmult_base(&point, sec.data());
ge_p3_tobytes(pub.data(), &point);
return true;
}
@ -209,14 +204,14 @@ namespace crypto {
ge_p3 point;
ge_p2 point2;
ge_p1p1 point3;
assert(sc_check(&key2) == 0);
if (ge_frombytes_vartime(&point, &key1) != 0) {
assert(sc_check(key2.data()) == 0);
if (ge_frombytes_vartime(&point, key1.data()) != 0) {
return false;
}
ge_scalarmult(&point2, &unwrap(key2), &point);
ge_scalarmult(&point2, key2.data(), &point);
ge_mul8(&point3, &point2);
ge_p1p1_to_p2(&point2, &point3);
ge_tobytes(&derivation, &point2);
ge_tobytes(derivation.data(), &point2);
return true;
}
@ -228,7 +223,7 @@ namespace crypto {
char *end = buf.output_index;
buf.derivation = derivation;
tools::write_varint(end, output_index);
hash_to_scalar(&buf, end - reinterpret_cast<char *>(&buf), res);
res = hash_to_scalar(&buf, end - reinterpret_cast<char *>(&buf));
}
bool derive_public_key(const key_derivation &derivation, size_t output_index,
@ -239,24 +234,24 @@ namespace crypto {
ge_cached point3;
ge_p1p1 point4;
ge_p2 point5;
if (ge_frombytes_vartime(&point1, &base) != 0) {
if (ge_frombytes_vartime(&point1, base.data()) != 0) {
return false;
}
derivation_to_scalar(derivation, output_index, scalar);
ge_scalarmult_base(&point2, &scalar);
ge_scalarmult_base(&point2, scalar.data());
ge_p3_to_cached(&point3, &point2);
ge_add(&point4, &point1, &point3);
ge_p1p1_to_p2(&point5, &point4);
ge_tobytes(&derived_key, &point5);
ge_tobytes(derived_key.data(), &point5);
return true;
}
void derive_secret_key(const key_derivation &derivation, size_t output_index,
const secret_key &base, secret_key &derived_key) {
ec_scalar scalar;
assert(sc_check(&base) == 0);
assert(sc_check(base.data()) == 0);
derivation_to_scalar(derivation, output_index, scalar);
sc_add(&unwrap(derived_key), &unwrap(base), &scalar);
sc_add(derived_key.data(), base.data(), scalar.data());
}
bool derive_subaddress_public_key(const public_key &out_key, const key_derivation &derivation, std::size_t output_index, public_key &derived_key) {
@ -266,15 +261,15 @@ namespace crypto {
ge_cached point3;
ge_p1p1 point4;
ge_p2 point5;
if (ge_frombytes_vartime(&point1, &out_key) != 0) {
if (ge_frombytes_vartime(&point1, out_key.data()) != 0) {
return false;
}
derivation_to_scalar(derivation, output_index, scalar);
ge_scalarmult_base(&point2, &scalar);
ge_scalarmult_base(&point2, scalar.data());
ge_p3_to_cached(&point3, &point2);
ge_sub(&point4, &point1, &point3);
ge_p1p1_to_p2(&point5, &point4);
ge_tobytes(&derived_key, &point5);
ge_tobytes(derived_key.data(), &point5);
return true;
}
@ -299,56 +294,64 @@ namespace crypto {
{
ge_p3 t;
public_key t2;
assert(sc_check(&sec) == 0);
ge_scalarmult_base(&t, &sec);
ge_p3_tobytes(&t2, &t);
assert(sc_check(sec.data()) == 0);
ge_scalarmult_base(&t, sec.data());
ge_p3_tobytes(t2.data(), &t);
assert(pub == t2);
}
#endif
buf.h = prefix_hash;
buf.key = pub;
signature sig;
try_again:
random_scalar(k);
ge_scalarmult_base(&tmp3, &k);
ge_p3_tobytes(&buf.comm, &tmp3);
hash_to_scalar(&buf, sizeof(s_comm), sig.c);
if (!sc_isnonzero((const unsigned char*)sig.c.data))
goto try_again;
sc_mulsub(&sig.r, &sig.c, &unwrap(sec), &k);
if (!sc_isnonzero((const unsigned char*)sig.r.data))
goto try_again;
memwipe(&k, sizeof(k));
return sig;
while (true)
{
random_scalar(k);
ge_scalarmult_base(&tmp3, k.data());
ge_p3_tobytes(buf.comm.data(), &tmp3);
sig.c(hash_to_scalar(&buf, sizeof(s_comm)));
if (!sc_isnonzero(sig.c()))
continue;
sc_mulsub(sig.r(), sig.c(), sec.data(), k.data());
if (!sc_isnonzero(sig.r()))
continue;
memwipe(k.data(), k.size());
return sig;
}
}
void generate_signature(const hash &prefix_hash, const public_key &pub, const secret_key &sec, signature &sig) {
sig = generate_signature(prefix_hash, pub, sec);
}
static constexpr ec_point infinity = {{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}};
static constexpr ec_point get_infinity() {
ec_point inf{};
inf.data_[0] = 1;
return inf;
}
constexpr ec_point infinity = get_infinity();
bool check_signature(const hash &prefix_hash, const public_key &pub, const signature &sig) {
ge_p2 tmp2;
ge_p3 tmp3;
ec_scalar c;
s_comm buf;
assert(check_key(pub));
buf.h = prefix_hash;
buf.key = pub;
if (ge_frombytes_vartime(&tmp3, &pub) != 0) {
if (ge_frombytes_vartime(&tmp3, pub.data()) != 0) {
return false;
}
if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0 || !sc_isnonzero(&sig.c)) {
if (sc_check(sig.c()) != 0 || sc_check(sig.r()) != 0 || !sc_isnonzero(sig.c())) {
return false;
}
ge_double_scalarmult_base_vartime(&tmp2, &sig.c, &tmp3, &sig.r); // tmp2 = sig.c A + sig.r G
ge_tobytes(&buf.comm, &tmp2);
if (memcmp(&buf.comm, &infinity, 32) == 0)
ge_double_scalarmult_base_vartime(&tmp2, sig.c(), &tmp3, sig.r()); // tmp2 = sig.c A + sig.r G
ge_tobytes(buf.comm.data(), &tmp2);
if (memcmp(buf.comm.data(), infinity.data(), 32) == 0)
return false;
hash_to_scalar(&buf, sizeof(s_comm), c);
sc_sub(&c, &c, &sig.c);
return sc_isnonzero(&c) == 0;
ec_scalar c = hash_to_scalar(&buf, sizeof(s_comm));
sc_sub(c.data(), c.data(), sig.c());
return sc_isnonzero(c.data()) == 0;
}
void generate_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const std::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) {
@ -357,33 +360,33 @@ namespace crypto {
ge_p3 A_p3;
ge_p3 B_p3;
ge_p3 D_p3;
if (ge_frombytes_vartime(&R_p3, &R) != 0) throw std::runtime_error("tx pubkey is invalid");
if (ge_frombytes_vartime(&A_p3, &A) != 0) throw std::runtime_error("recipient view pubkey is invalid");
if (B && ge_frombytes_vartime(&B_p3, &*B) != 0) throw std::runtime_error("recipient spend pubkey is invalid");
if (ge_frombytes_vartime(&D_p3, &D) != 0) throw std::runtime_error("key derivation is invalid");
if (ge_frombytes_vartime(&R_p3, R.data()) != 0) throw std::runtime_error("tx pubkey is invalid");
if (ge_frombytes_vartime(&A_p3, A.data()) != 0) throw std::runtime_error("recipient view pubkey is invalid");
if (B && ge_frombytes_vartime(&B_p3, B->data()) != 0) throw std::runtime_error("recipient spend pubkey is invalid");
if (ge_frombytes_vartime(&D_p3, D.data()) != 0) throw std::runtime_error("key derivation is invalid");
#if !defined(NDEBUG)
{
assert(sc_check(&r) == 0);
assert(sc_check(r.data()) == 0);
// check R == r*G or R == r*B
public_key dbg_R;
if (B)
{
ge_p2 dbg_R_p2;
ge_scalarmult(&dbg_R_p2, &r, &B_p3);
ge_tobytes(&dbg_R, &dbg_R_p2);
ge_scalarmult(&dbg_R_p2, r.data(), &B_p3);
ge_tobytes(dbg_R.data(), &dbg_R_p2);
}
else
{
ge_p3 dbg_R_p3;
ge_scalarmult_base(&dbg_R_p3, &r);
ge_p3_tobytes(&dbg_R, &dbg_R_p3);
ge_scalarmult_base(&dbg_R_p3, r.data());
ge_p3_tobytes(dbg_R.data(), &dbg_R_p3);
}
assert(R == dbg_R);
// check D == r*A
ge_p2 dbg_D_p2;
ge_scalarmult(&dbg_D_p2, &r, &A_p3);
ge_scalarmult(&dbg_D_p2, r.data(), &A_p3);
public_key dbg_D;
ge_tobytes(&dbg_D, &dbg_D_p2);
ge_tobytes(dbg_D.data(), &dbg_D_p2);
assert(D == dbg_D);
}
#endif
@ -399,29 +402,29 @@ namespace crypto {
{
// compute X = k*B
ge_p2 X_p2;
ge_scalarmult(&X_p2, &k, &B_p3);
ge_tobytes(&buf.X, &X_p2);
ge_scalarmult(&X_p2, k.data(), &B_p3);
ge_tobytes(buf.X.data(), &X_p2);
}
else
{
// compute X = k*G
ge_p3 X_p3;
ge_scalarmult_base(&X_p3, &k);
ge_p3_tobytes(&buf.X, &X_p3);
ge_scalarmult_base(&X_p3, k.data());
ge_p3_tobytes(buf.X.data(), &X_p3);
}
// compute Y = k*A
ge_p2 Y_p2;
ge_scalarmult(&Y_p2, &k, &A_p3);
ge_tobytes(&buf.Y, &Y_p2);
ge_scalarmult(&Y_p2, k.data(), &A_p3);
ge_tobytes(buf.Y.data(), &Y_p2);
// sig.c = Hs(Msg || D || X || Y)
hash_to_scalar(&buf, sizeof(buf), sig.c);
sig.c(hash_to_scalar(&buf, sizeof(buf)));
// sig.r = k - sig.c*r
sc_mulsub(&sig.r, &sig.c, &unwrap(r), &k);
sc_mulsub(sig.r(), sig.c(), r.data(), k.data());
memwipe(&k, sizeof(k));
memwipe(k.data(), k.size());
}
bool check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const std::optional<public_key> &B, const public_key &D, const signature &sig) {
@ -430,20 +433,20 @@ namespace crypto {
ge_p3 A_p3;
ge_p3 B_p3;
ge_p3 D_p3;
if (ge_frombytes_vartime(&R_p3, &R) != 0) return false;
if (ge_frombytes_vartime(&A_p3, &A) != 0) return false;
if (B && ge_frombytes_vartime(&B_p3, &*B) != 0) return false;
if (ge_frombytes_vartime(&D_p3, &D) != 0) return false;
if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0) return false;
if (ge_frombytes_vartime(&R_p3, R.data()) != 0) return false;
if (ge_frombytes_vartime(&A_p3, A.data()) != 0) return false;
if (B && ge_frombytes_vartime(&B_p3, B->data()) != 0) return false;
if (ge_frombytes_vartime(&D_p3, D.data()) != 0) return false;
if (sc_check(sig.c()) != 0 || sc_check(sig.r()) != 0) return false;
// compute sig.c*R
ge_p3 cR_p3;
{
ge_p2 cR_p2;
ge_scalarmult(&cR_p2, &sig.c, &R_p3);
ge_scalarmult(&cR_p2, sig.c(), &R_p3);
public_key cR;
ge_tobytes(&cR, &cR_p2);
if (ge_frombytes_vartime(&cR_p3, &cR) != 0) return false;
ge_tobytes(cR.data(), &cR_p2);
if (ge_frombytes_vartime(&cR_p3, cR.data()) != 0) return false;
}
ge_p1p1 X_p1p1;
@ -451,11 +454,11 @@ namespace crypto {
{
// compute X = sig.c*R + sig.r*B
ge_p2 rB_p2;
ge_scalarmult(&rB_p2, &sig.r, &B_p3);
ge_scalarmult(&rB_p2, sig.r(), &B_p3);
public_key rB;
ge_tobytes(&rB, &rB_p2);
ge_tobytes(rB.data(), &rB_p2);
ge_p3 rB_p3;
if (ge_frombytes_vartime(&rB_p3, &rB) != 0) return false;
if (ge_frombytes_vartime(&rB_p3, rB.data()) != 0) return false;
ge_cached rB_cached;
ge_p3_to_cached(&rB_cached, &rB_p3);
ge_add(&X_p1p1, &cR_p3, &rB_cached);
@ -464,7 +467,7 @@ namespace crypto {
{
// compute X = sig.c*R + sig.r*G
ge_p3 rG_p3;
ge_scalarmult_base(&rG_p3, &sig.r);
ge_scalarmult_base(&rG_p3, sig.r());
ge_cached rG_cached;
ge_p3_to_cached(&rG_cached, &rG_p3);
ge_add(&X_p1p1, &cR_p3, &rG_cached);
@ -474,21 +477,21 @@ namespace crypto {
// compute sig.c*D
ge_p2 cD_p2;
ge_scalarmult(&cD_p2, &sig.c, &D_p3);
ge_scalarmult(&cD_p2, sig.c(), &D_p3);
// compute sig.r*A
ge_p2 rA_p2;
ge_scalarmult(&rA_p2, &sig.r, &A_p3);
ge_scalarmult(&rA_p2, sig.r(), &A_p3);
// compute Y = sig.c*D + sig.r*A
public_key cD;
public_key rA;
ge_tobytes(&cD, &cD_p2);
ge_tobytes(&rA, &rA_p2);
ge_tobytes(cD.data(), &cD_p2);
ge_tobytes(rA.data(), &rA_p2);
ge_p3 cD_p3;
ge_p3 rA_p3;
if (ge_frombytes_vartime(&cD_p3, &cD) != 0) return false;
if (ge_frombytes_vartime(&rA_p3, &rA) != 0) return false;
if (ge_frombytes_vartime(&cD_p3, cD.data()) != 0) return false;
if (ge_frombytes_vartime(&rA_p3, rA.data()) != 0) return false;
ge_cached rA_cached;
ge_p3_to_cached(&rA_cached, &rA_p3);
ge_p1p1 Y_p1p1;
@ -500,14 +503,13 @@ namespace crypto {
s_comm_2 buf;
buf.msg = prefix_hash;
buf.D = D;
ge_tobytes(&buf.X, &X_p2);
ge_tobytes(&buf.Y, &Y_p2);
ec_scalar c2;
hash_to_scalar(&buf, sizeof(s_comm_2), c2);
ge_tobytes(buf.X.data(), &X_p2);
ge_tobytes(buf.Y.data(), &Y_p2);
ec_scalar c2 = hash_to_scalar(&buf, sizeof(s_comm_2));
// test if c2 == sig.c
sc_sub(&c2, &c2, &sig.c);
return sc_isnonzero(&c2) == 0;
sc_sub(c2.data(), c2.data(), sig.c());
return sc_isnonzero(c2.data()) == 0;
}
static void hash_to_ec(const public_key &key, ge_p3 &res) {
@ -523,10 +525,10 @@ namespace crypto {
void generate_key_image(const public_key &pub, const secret_key &sec, key_image &image) {
ge_p3 point;
ge_p2 point2;
assert(sc_check(&sec) == 0);
assert(sc_check(sec.data()) == 0);
hash_to_ec(pub, point);
ge_scalarmult(&point2, &unwrap(sec), &point);
ge_tobytes(&image, &point2);
ge_scalarmult(&point2, sec.data(), &point);
ge_tobytes(image.data(), &point2);
}
struct rs_comm {
@ -542,8 +544,8 @@ namespace crypto {
static_assert(sizeof(ab[0]) == 64); // Ensure no padding
keccak_update(&state, reinterpret_cast<const uint8_t*>(ab.data()), 64*ab.size());
ec_scalar result;
keccak_finish(&state, reinterpret_cast<uint8_t*>(&result));
sc_reduce32(&result);
keccak_finish(&state, result.data());
sc_reduce32(result.data());
return result;
};
};
@ -563,9 +565,9 @@ namespace crypto {
ge_p3 t;
public_key t2;
key_image t3;
assert(sc_check(&sec) == 0);
ge_scalarmult_base(&t, &sec);
ge_p3_tobytes(&t2, &t);
assert(sc_check(sec.data()) == 0);
ge_scalarmult_base(&t, sec.data());
ge_p3_tobytes(t2.data(), &t);
assert(*pubs[sec_index] == t2);
generate_key_image(*pubs[sec_index], sec, t3);
assert(image == t3);
@ -575,13 +577,12 @@ namespace crypto {
}
#endif
ge_p3 image_unp; // I
if (ge_frombytes_vartime(&image_unp, &image) != 0) {
if (ge_frombytes_vartime(&image_unp, image.data()) != 0) {
local_abort("invalid key image");
}
ge_dsmp image_pre;
ge_dsm_precomp(image_pre, &image_unp);
ec_scalar sum;
sc_0(&sum); // will be sum of cj, j≠s
ec_scalar sum = null<ec_scalar>; // will be sum of cj, j≠s
rs_comm rs{prefix_hash, pubs.size()};
ec_scalar qs;
@ -590,32 +591,32 @@ namespace crypto {
ge_p3 tmp3;
if (i == sec_index) { // this is the true key image
random_scalar(qs); // qs = random
ge_scalarmult_base(&tmp3, &qs); // Ls = qs G
ge_p3_tobytes(&rs.ab[i].first, &tmp3);
ge_scalarmult_base(&tmp3, qs.data()); // Ls = qs G
ge_p3_tobytes(rs.ab[i].first.data(), &tmp3);
hash_to_ec(*pubs[i], tmp3); // Hp(Ps)
ge_scalarmult(&tmp2, &qs, &tmp3); // Rs = qs Hp(Ps)
ge_tobytes(&rs.ab[i].second, &tmp2);
ge_scalarmult(&tmp2, qs.data(), &tmp3); // Rs = qs Hp(Ps)
ge_tobytes(rs.ab[i].second.data(), &tmp2);
// We don't set ci, ri yet because we first need the sum of all the other cj's/rj's
} else {
random_scalar(sig[i].c); // ci = wi = random
random_scalar(sig[i].r); // ri = qi = random
if (ge_frombytes_vartime(&tmp3, &*pubs[i]) != 0) {
memwipe(&qs, sizeof(qs));
random_scalar(sig[i].c()); // ci = wi = random
random_scalar(sig[i].r()); // ri = qi = random
if (ge_frombytes_vartime(&tmp3, pubs[i]->data()) != 0) {
memwipe(qs.data(), qs.size());
local_abort("invalid pubkey");
}
ge_double_scalarmult_base_vartime(&tmp2, &sig[i].c, &tmp3, &sig[i].r); // Li = cj Pj + rj G = qj G + wj Pj
ge_tobytes(&rs.ab[i].first, &tmp2);
ge_double_scalarmult_base_vartime(&tmp2, sig[i].c(), &tmp3, sig[i].r()); // Li = cj Pj + rj G = qj G + wj Pj
ge_tobytes(rs.ab[i].first.data(), &tmp2);
hash_to_ec(*pubs[i], tmp3); // Hp(Pj)
ge_double_scalarmult_precomp_vartime(&tmp2, &sig[i].r, &tmp3, &sig[i].c, image_pre); // Ri = qj Hp(Pj) + wj I
ge_tobytes(&rs.ab[i].second, &tmp2);
sc_add(&sum, &sum, &sig[i].c);
ge_double_scalarmult_precomp_vartime(&tmp2, sig[i].r(), &tmp3, sig[i].c(), image_pre); // Ri = qj Hp(Pj) + wj I
ge_tobytes(rs.ab[i].second.data(), &tmp2);
sc_add(sum.data(), sum.data(), sig[i].c());
}
}
ec_scalar c = rs.hash_to_scalar(); // c = Hs(prefix_hash || L0 || ... || L{n-1} || R0 || ... || R{n-1})
sc_sub(&sig[sec_index].c, &c, &sum); // cs = c - sum(ci, i≠s) = c - sum(wi)
sc_mulsub(&sig[sec_index].r, &sig[sec_index].c, &unwrap(sec), &qs); // rs = qs - cs*x
sc_sub(sig[sec_index].c(), c.data(), sum.data()); // cs = c - sum(ci, i≠s) = c - sum(wi)
sc_mulsub(sig[sec_index].r(), sig[sec_index].c(), sec.data(), qs.data()); // rs = qs - cs*x
memwipe(&qs, sizeof(qs));
memwipe(qs.data(), qs.size());
}
bool check_ring_signature(
@ -629,34 +630,33 @@ namespace crypto {
}
#endif
ge_p3 image_unp;
if (ge_frombytes_vartime(&image_unp, &image) != 0) {
if (ge_frombytes_vartime(&image_unp, image.data()) != 0) {
return false;
}
ge_dsmp image_pre;
ge_dsm_precomp(image_pre, &image_unp);
ec_scalar sum;
sc_0(&sum);
ec_scalar sum = null<ec_scalar>;
rs_comm rs{prefix_hash, pubs.size()};
for (size_t i = 0; i < pubs.size(); i++) {
ge_p2 tmp2;
ge_p3 tmp3;
if (sc_check(&sig[i].c) != 0 || sc_check(&sig[i].r) != 0) {
if (sc_check(sig[i].c()) != 0 || sc_check(sig[i].r()) != 0) {
return false;
}
if (ge_frombytes_vartime(&tmp3, &*pubs[i]) != 0) {
if (ge_frombytes_vartime(&tmp3, pubs[i]->data()) != 0) {
return false;
}
ge_double_scalarmult_base_vartime(&tmp2, &sig[i].c, &tmp3, &sig[i].r);
ge_tobytes(&rs.ab[i].first, &tmp2);
ge_double_scalarmult_base_vartime(&tmp2, sig[i].c(), &tmp3, sig[i].r());
ge_tobytes(rs.ab[i].first.data(), &tmp2);
hash_to_ec(*pubs[i], tmp3);
ge_double_scalarmult_precomp_vartime(&tmp2, &sig[i].r, &tmp3, &sig[i].c, image_pre);
ge_tobytes(&rs.ab[i].second, &tmp2);
sc_add(&sum, &sum, &sig[i].c);
ge_double_scalarmult_precomp_vartime(&tmp2, sig[i].r(), &tmp3, sig[i].c(), image_pre);
ge_tobytes(rs.ab[i].second.data(), &tmp2);
sc_add(sum.data(), sum.data(), sig[i].c());
}
ec_scalar h = rs.hash_to_scalar();
sc_sub(&h, &h, &sum);
return sc_isnonzero(&h) == 0;
sc_sub(h.data(), h.data(), sum.data());
return sc_isnonzero(h.data()) == 0;
}
void generate_key_image_signature(
@ -669,19 +669,19 @@ namespace crypto {
rs_comm rs{reinterpret_cast<const hash&>(image), 1};
ge_p3 tmp3;
ge_scalarmult_base(&tmp3, &k); // L = kG
ge_p3_tobytes(&rs.ab[0].first, &tmp3); // store L
ge_scalarmult_base(&tmp3, k.data()); // L = kG
ge_p3_tobytes(rs.ab[0].first.data(), &tmp3); // store L
hash_to_ec(pub, tmp3); // H(A)
ge_p2 tmp2;
ge_scalarmult(&tmp2, &k, &tmp3); // R = kH(A)
ge_tobytes(&rs.ab[0].second, &tmp2); // store R
ge_scalarmult(&tmp2, k.data(), &tmp3); // R = kH(A)
ge_tobytes(rs.ab[0].second.data(), &tmp2); // store R
sig.c = rs.hash_to_scalar(); // c = H(I || L || R) = H(I || kG || kH(A))
sc_mulsub(&sig.r, &sig.c, &unwrap(sec), &k); // r = k - ac = k - aH(I || kG || kH(A))
sig.c(rs.hash_to_scalar()); // c = H(I || L || R) = H(I || kG || kH(A))
sc_mulsub(sig.r(), sig.c(), sec.data(), k.data()); // r = k - ac = k - aH(I || kG || kH(A))
memwipe(&k, sizeof(k));
memwipe(k.data(), k.size());
}
bool check_key_image_signature(
@ -691,14 +691,14 @@ namespace crypto {
assert(check_key(pub));
ge_p3 image_unp;
if (ge_frombytes_vartime(&image_unp, &image) != 0 || sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0)
if (ge_frombytes_vartime(&image_unp, image.data()) != 0 || sc_check(sig.c()) != 0 || sc_check(sig.r()) != 0)
return false;
ge_dsmp image_pre;
ge_dsm_precomp(image_pre, &image_unp);
rs_comm rs{reinterpret_cast<const hash&>(image), 1};
ge_p3 tmp3;
if (ge_frombytes_vartime(&tmp3, &pub) != 0)
if (ge_frombytes_vartime(&tmp3, pub.data()) != 0)
return false;
ge_p2 tmp2;
@ -706,8 +706,8 @@ namespace crypto {
// The signature r was constructed as r = k - ac, so:
// k = ac + r
// kG = cA + rG = L
ge_double_scalarmult_base_vartime(&tmp2, &sig.c, &tmp3, &sig.r); // L = cA + rG
ge_tobytes(&rs.ab[0].first, &tmp2); // store L
ge_double_scalarmult_base_vartime(&tmp2, sig.c(), &tmp3, sig.r()); // L = cA + rG
ge_tobytes(rs.ab[0].first.data(), &tmp2); // store L
// Step two: reconstruct the signer's R = kH(A)
// The signature r was constructed as r = k - ac, so:
@ -715,14 +715,14 @@ namespace crypto {
// and since aH(A) == I (the key image, by definition):
// kH(A) = rH(A) + cI = R
hash_to_ec(pub, tmp3); // H(A)
ge_double_scalarmult_precomp_vartime(&tmp2, &sig.r, &tmp3, &sig.c, image_pre); // R = rH(A) + cI
ge_tobytes(&rs.ab[0].second, &tmp2); // store R
ge_double_scalarmult_precomp_vartime(&tmp2, sig.r(), &tmp3, sig.c(), image_pre); // R = rH(A) + cI
ge_tobytes(rs.ab[0].second.data(), &tmp2); // store R
// Now we can calculate our own H(I || L || R), and compare it to the signature's c (which was
// set to the signer's H(I || L || R) calculation).
ec_scalar h = rs.hash_to_scalar();
sc_sub(&h, &h, &sig.c);
return sc_isnonzero(&h) == 0;
sc_sub(h.data(), h.data(), sig.c());
return sc_isnonzero(h.data()) == 0;
}
}

133
src/crypto/crypto.h
View File

@ -39,101 +39,76 @@
#include "epee/memwipe.h"
#include "epee/mlocker.h"
#include "generic-ops.h"
#include "common/hex.h"
#include "hash.h"
#include "base.h"
extern "C" {
#include "random.h"
}
namespace crypto {
extern "C" {
#include "random.h"
}
struct alignas(size_t) ec_point {
char data[32];
// Returns true if non-null, i.e. not 0.
operator bool() const { static constexpr char null[32] = {0}; return memcmp(data, null, sizeof(data)); }
struct ec_point : bytes<32, true> {
// Returns true if non-null, i.e. not all 0.
explicit operator bool() const { return data_ != null<ec_point>.data_; }
};
struct alignas(size_t) ec_scalar {
char data[32];
struct ec_scalar : bytes<32> {
// constant-time (via libsodium)
bool operator==(const ec_scalar& x) const;
bool operator!=(const ec_scalar& x) const { return !(*this == x); }
// constant-time returns true if not all 0.
explicit operator bool() const;
};
struct public_key : ec_point {};
using secret_key = epee::mlocked<tools::scrubbed<ec_scalar>>;
struct secret_key_ : ec_scalar {};
using secret_key = epee::mlocked<tools::scrubbed<secret_key_>>;
struct public_keyV {
std::vector<public_key> keys;
int rows;
};
struct secret_keyV {
std::vector<secret_key> keys;
int rows;
};
struct public_keyM {
int cols;
int rows;
std::vector<secret_keyV> column_vectors;
};
template <> inline const secret_key null<secret_key>{};
struct key_derivation: ec_point {};
struct key_image: ec_point {};
struct signature {
ec_scalar c, r;
struct signature : bytes<64, true> {
// Returns or sets the "c" part of the signature bytes
unsigned char* c() { return data(); }
const unsigned char* c() const { return data(); }
void c(const ec_scalar& c) { std::copy(c.data(), c.data() + c.size(), data()); }
// Returns or sets the "r" part of the signature bytes
unsigned char* r() { return data() + 32; }
const unsigned char* r() const { return data() + 32; }
void r(const ec_scalar& r) { std::copy(r.data(), r.data() + r.size(), data()); }
// Returns true if non-null, i.e. not 0.
operator bool() const { static constexpr char null[64] = {0}; return memcmp(this, null, sizeof(null)); }
explicit operator bool() const { return data_ != null<signature>.data_; }
};
// The sizes below are all provided by sodium.h, but we don't want to depend on it here; we check
// that they agree with the actual constants from sodium.h when compiling cryptonote_core.cpp.
struct alignas(size_t) ed25519_public_key {
unsigned char data[32]; // 32 = crypto_sign_ed25519_PUBLICKEYBYTES
static constexpr ed25519_public_key null() { return {0}; }
/// Returns true if non-null
operator bool() const { return memcmp(data, null().data, sizeof(data)); }
};
struct ed25519_public_key : ec_point {};
struct alignas(size_t) ed25519_secret_key_ {
// 64 = crypto_sign_ed25519_SECRETKEYBYTES (but we don't depend on libsodium header here)
unsigned char data[64];
};
// 64 = crypto_sign_ed25519_SECRETKEYBYTES (but we don't depend on libsodium header here)
struct ed25519_secret_key_ : bytes<64> {};
using ed25519_secret_key = epee::mlocked<tools::scrubbed<ed25519_secret_key_>>;
struct alignas(size_t) ed25519_signature {
unsigned char data[64]; // 64 = crypto_sign_BYTES
static constexpr ed25519_signature null() { return {0}; }
struct ed25519_signature : bytes<64, true> {
// Returns true if non-null, i.e. not 0.
operator bool() const { auto z = null(); return memcmp(this, &z, sizeof(z)); }
explicit operator bool() const { return data_ != null<ed25519_signature>.data_; }
};
struct alignas(size_t) x25519_public_key {
unsigned char data[32]; // crypto_scalarmult_curve25519_BYTES
static constexpr x25519_public_key null() { return {0}; }
/// Returns true if non-null
operator bool() const { return memcmp(data, null().data, sizeof(data)); }
};
struct x25519_public_key : ec_point {};
struct alignas(size_t) x25519_secret_key_ {
unsigned char data[32]; // crypto_scalarmult_curve25519_BYTES
};
struct x25519_secret_key_ : bytes<32> {};
using x25519_secret_key = epee::mlocked<tools::scrubbed<x25519_secret_key_>>;
void hash_to_scalar(const void *data, size_t length, ec_scalar &res);
ec_scalar hash_to_scalar(const void* data, size_t length);
void random_scalar(unsigned char* bytes);
void random_scalar(ec_scalar& res);
ec_scalar random_scalar();
static_assert(sizeof(ec_point) == 32 && sizeof(ec_scalar) == 32 &&
sizeof(public_key) == 32 && sizeof(secret_key) == 32 &&
sizeof(key_derivation) == 32 && sizeof(key_image) == 32 &&
sizeof(signature) == 64, "Invalid structure size");
void generate_random_bytes_thread_safe(size_t N, uint8_t *bytes);
void add_extra_entropy_thread_safe(const void *ptr, size_t bytes);
@ -284,34 +259,12 @@ namespace crypto {
const public_key& pub,
const signature& sig);
inline std::ostream &operator <<(std::ostream &o, const crypto::public_key &v) {
return o << '<' << tools::type_to_hex(v) << '>';
}
inline std::ostream &operator <<(std::ostream &o, const crypto::secret_key &v) {
return o << '<' << tools::type_to_hex(v) << '>';
}
inline std::ostream &operator <<(std::ostream &o, const crypto::key_derivation &v) {
return o << '<' << tools::type_to_hex(v) << '>';
}
inline std::ostream &operator <<(std::ostream &o, const crypto::key_image &v) {
return o << '<' << tools::type_to_hex(v) << '>';
}
inline std::ostream &operator <<(std::ostream &o, const crypto::signature &v) {
return o << '<' << tools::type_to_hex(v) << '>';
}
inline std::ostream &operator <<(std::ostream &o, const crypto::ed25519_public_key &v) {
return o << '<' << tools::type_to_hex(v) << '>';
}
inline std::ostream &operator <<(std::ostream &o, const crypto::x25519_public_key &v) {
return o << '<' << tools::type_to_hex(v) << '>';
}
constexpr inline crypto::public_key null_pkey{};
const inline crypto::secret_key null_skey{};
}
CRYPTO_MAKE_HASHABLE(public_key)
CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(secret_key)
CRYPTO_MAKE_HASHABLE(key_image)
CRYPTO_MAKE_HASHABLE(signature)
CRYPTO_MAKE_HASHABLE(ed25519_public_key)
CRYPTO_MAKE_HASHABLE(x25519_public_key)
template <> struct std::hash<crypto::ec_point> : crypto::raw_hasher<crypto::ec_point> {};
template <> struct std::hash<crypto::public_key> : crypto::raw_hasher<crypto::public_key> {};
template <> struct std::hash<crypto::key_image> : crypto::raw_hasher<crypto::key_image> {};
template <> struct std::hash<crypto::signature> : crypto::raw_hasher<crypto::signature> {};
template <> struct std::hash<crypto::ed25519_public_key> : crypto::raw_hasher<crypto::ed25519_public_key> {};
template <> struct std::hash<crypto::x25519_public_key> : crypto::raw_hasher<crypto::x25519_public_key> {};
template <> struct std::hash<crypto::ed25519_signature> : crypto::raw_hasher<crypto::ed25519_signature> {};

81
src/crypto/generic-ops.h
View File

@ -1,81 +0,0 @@
// Copyright (c) 2014-2019, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
#pragma once
#include <cstddef>
#include <cstring>
#include <functional>
#include <sodium/crypto_verify_32.h>
#define CRYPTO_MAKE_COMPARABLE(type) \
namespace crypto { \
inline bool operator==(const type &_v1, const type &_v2) { \
return !memcmp(&_v1, &_v2, sizeof(_v1)); \
} \
inline bool operator!=(const type &_v1, const type &_v2) { \
return !operator==(_v1, _v2); \
} \
inline bool operator<(const type &_v1, const type &_v2) { \
return memcmp(&_v1, &_v2, sizeof(_v1)) < 0; \
} \
}
#define CRYPTO_MAKE_COMPARABLE_CONSTANT_TIME(type) \
namespace crypto { \
inline bool operator==(const type &_v1, const type &_v2) { \
static_assert(sizeof(_v1) == 32, "constant time comparison is only implenmted for 32 bytes"); \
return crypto_verify_32((const unsigned char*)&_v1, (const unsigned char*)&_v2) == 0; \
} \
inline bool operator!=(const type &_v1, const type &_v2) { \
return !operator==(_v1, _v2); \
} \
}
#define CRYPTO_DEFINE_HASH_FUNCTIONS(type) \
namespace std { \
template<> \
struct hash<crypto::type> { \
static_assert(sizeof(crypto::type) >= sizeof(std::size_t) && alignof(crypto::type) >= alignof(std::size_t), \
"Size and alignment of " #type " must be at least that of size_t"); \
std::size_t operator()(const crypto::type &_v) const { \
return reinterpret_cast<const std::size_t &>(_v); \
} \
}; \
}
#define CRYPTO_MAKE_HASHABLE(type) \
CRYPTO_MAKE_COMPARABLE(type) \
CRYPTO_DEFINE_HASH_FUNCTIONS(type)
#define CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(type) \
CRYPTO_MAKE_COMPARABLE_CONSTANT_TIME(type) \
CRYPTO_DEFINE_HASH_FUNCTIONS(type)

4
src/crypto/hash-extra-jh.c
View File

@ -35,7 +35,7 @@
#include "jh.h"
#include "hash-ops.h"
void hash_extra_jh(const void *data, size_t length, char *hash) {
int r = jh_hash(HASH_SIZE * 8, data, 8 * length, (uint8_t*)hash);
void hash_extra_jh(const void *data, size_t length, unsigned char *hash) {
int r = jh_hash(HASH_SIZE * 8, data, 8 * length, hash);
assert(SUCCESS == r);
}

Some files were not shown because too many files have changed in this diff Show More