2019-04-12 06:36:43 +02:00
// Copyright (c) 2014-2019, The Monero Project
2018-04-10 06:49:20 +02:00
// Copyright (c) 2018, The Loki Project
2015-12-14 05:54:39 +01:00
//
2014-07-23 15:03:52 +02:00
// All rights reserved.
2015-12-14 05:54:39 +01:00
//
2014-07-23 15:03:52 +02:00
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
2015-12-14 05:54:39 +01:00
//
2014-07-23 15:03:52 +02:00
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
2015-12-14 05:54:39 +01:00
//
2014-07-23 15:03:52 +02:00
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
2015-12-14 05:54:39 +01:00
//
2014-07-23 15:03:52 +02:00
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
2015-12-14 05:54:39 +01:00
//
2014-07-23 15:03:52 +02:00
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2015-12-14 05:54:39 +01:00
//
2014-07-23 15:03:52 +02:00
// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
2014-03-03 23:07:58 +01:00
2014-04-02 18:00:17 +02:00
# include <algorithm>
2014-03-03 23:07:58 +01:00
# include <unordered_set>
2014-04-02 18:00:17 +02:00
# include <vector>
2014-03-03 23:07:58 +01:00
# include "tx_pool.h"
2017-01-26 16:07:23 +01:00
# include "cryptonote_tx_utils.h"
# include "cryptonote_basic/cryptonote_boost_serialization.h"
2019-07-04 09:25:02 +02:00
# include "cryptonote_core/service_node_list.h"
2014-03-03 23:07:58 +01:00
# include "cryptonote_config.h"
2014-10-28 04:44:45 +01:00
# include "blockchain.h"
2019-12-10 19:15:00 +01:00
# include "blockchain_db/locked_txn.h"
2017-05-14 15:06:55 +02:00
# include "blockchain_db/blockchain_db.h"
2014-03-03 23:07:58 +01:00
# include "common/boost_serialization_helper.h"
2019-12-12 04:11:07 +01:00
# include "common/lock.h"
2020-10-23 22:32:28 +02:00
# include "common/hex.h"
2022-04-15 04:34:09 +02:00
# include "common/median.h"
2020-10-24 00:49:42 +02:00
# include "epee/int-util.h"
# include "epee/warnings.h"
2016-10-10 21:47:52 +02:00
# include "common/perf_timer.h"
2014-03-03 23:07:58 +01:00
# include "crypto/hash.h"
2021-01-04 04:19:42 +01:00
# undef OXEN_DEFAULT_LOG_CATEGORY
# define OXEN_DEFAULT_LOG_CATEGORY "txpool"
Change logging to easylogging++
This replaces the epee and data_loggers logging systems with
a single one, and also adds filename:line and explicit severity
levels. Categories may be defined, and logging severity set
by category (or set of categories). epee style 0-4 log level
maps to a sensible severity configuration. Log files now also
rotate when reaching 100 MB.
To select which logs to output, use the MONERO_LOGS environment
variable, with a comma separated list of categories (globs are
supported), with their requested severity level after a colon.
If a log matches more than one such setting, the last one in
the configuration string applies. A few examples:
This one is (mostly) silent, only outputting fatal errors:
MONERO_LOGS=*:FATAL
This one is very verbose:
MONERO_LOGS=*:TRACE
This one is totally silent (logwise):
MONERO_LOGS=""
This one outputs all errors and warnings, except for the
"verify" category, which prints just fatal errors (the verify
category is used for logs about incoming transactions and
blocks, and it is expected that some/many will fail to verify,
hence we don't want the spam):
MONERO_LOGS=*:WARNING,verify:FATAL
Log levels are, in decreasing order of priority:
FATAL, ERROR, WARNING, INFO, DEBUG, TRACE
Subcategories may be added using prefixes and globs. This
example will output net.p2p logs at the TRACE level, but all
other net* logs only at INFO:
MONERO_LOGS=*:ERROR,net*:INFO,net.p2p:TRACE
Logs which are intended for the user (which Monero was using
a lot through epee, but really isn't a nice way to go things)
should use the "global" category. There are a few helper macros
for using this category, eg: MGINFO("this shows up by default")
or MGINFO_RED("this is red"), to try to keep a similar look
and feel for now.
Existing epee log macros still exist, and map to the new log
levels, but since they're used as a "user facing" UI element
as much as a logging system, they often don't map well to log
severities (ie, a log level 0 log may be an error, or may be
something we want the user to see, such as an important info).
In those cases, I tried to use the new macros. In other cases,
I left the existing macros in. When modifying logs, it is
probably best to switch to the new macros with explicit levels.
The --log-level options and set_log commands now also accept
category settings, in addition to the epee style log levels.
2017-01-01 17:34:23 +01:00
2014-03-03 23:07:58 +01:00
DISABLE_VS_WARNINGS ( 4244 4345 4503 ) //'boost::foreach_detail_::or_' : decorated name length exceeded, name was truncated
2017-10-10 16:47:08 +02:00
using namespace crypto ;
2014-03-03 23:07:58 +01:00
namespace cryptonote
{
2014-07-17 17:02:05 +02:00
namespace
{
2016-03-30 17:55:54 +02:00
//TODO: constants such as these should at least be in the header,
// but probably somewhere more accessible to the rest of the
// codebase. As it stands, it is at best nontrivial to test
// whether or not changing these parameters (or adding new)
// will work correctly.
2015-11-21 01:26:48 +01:00
time_t const MIN_RELAY_TIME = ( 60 * 5 ) ; // only start re-relaying transactions after that many seconds
time_t const MAX_RELAY_TIME = ( 60 * 60 * 4 ) ; // at most that many seconds between resends
2017-01-13 00:39:24 +01:00
float const ACCEPT_THRESHOLD = 1.0f ;
2015-11-21 01:26:48 +01:00
// a kind of increasing backoff within min/max bounds
2017-05-14 15:06:55 +02:00
uint64_t get_relay_delay ( time_t now , time_t received )
2015-11-21 01:26:48 +01:00
{
time_t d = ( now - received + MIN_RELAY_TIME ) / MIN_RELAY_TIME * MIN_RELAY_TIME ;
if ( d > MAX_RELAY_TIME )
d = MAX_RELAY_TIME ;
return d ;
}
2017-01-06 21:34:37 +01:00
2018-07-18 23:24:53 +02:00
uint64_t get_transaction_weight_limit ( uint8_t version )
2017-03-14 20:28:38 +01:00
{
2018-10-09 11:11:37 +02:00
// from v10, bulletproofs, limit a tx to 50% of the minimum block weight
2019-01-25 04:15:52 +01:00
if ( version > = network_version_10_bulletproofs )
2018-07-18 23:24:53 +02:00
return get_min_block_weight ( version ) / 2 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE ;
else
return get_min_block_weight ( version ) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE ;
2017-03-14 20:28:38 +01:00
}
2014-07-17 17:02:05 +02:00
}
2015-01-26 06:36:09 +01:00
//---------------------------------------------------------------------------------
2019-10-27 23:47:19 +01:00
// warning: bchs is passed here uninitialized, so don't do anything but store it
2018-07-18 23:24:53 +02:00
tx_memory_pool : : tx_memory_pool ( Blockchain & bchs ) : m_blockchain ( bchs ) , m_txpool_max_weight ( DEFAULT_TXPOOL_MAX_WEIGHT ) , m_txpool_weight ( 0 ) , m_cookie ( 0 )
2014-03-03 23:07:58 +01:00
{
Service Node Deregister Part 5 (#89)
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* core, service_node_list: separated address from service node pubkey
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* Store service node lists for the duration of deregister lifetimes
* Quorum min/max bug, sort node list, fix node to test list
* Change quorum to store acc pub address, fix oob bug
* Code review for expiring votes, acc keys to pub_key, improve err msgs
* Add early out for is_deregistration_tx and protect against quorum changes
* Remove debug code, fix segfault
* Remove irrelevant check for tx v3 in blockchain, fix >= height for pruning quorum states
Incorrect assumption that a transaction can be kept in the chain if it could
eventually become invalid, because if it were the chain would be split and
eventually these transaction would be dropped. But also that we should not
override the pre-existing logic which handles this case anyway.
2018-07-18 04:42:47 +02:00
}
//---------------------------------------------------------------------------------
2019-10-27 23:26:38 +01:00
bool tx_memory_pool : : have_duplicated_non_standard_tx ( transaction const & tx , uint8_t hard_fork_version ) const
Service Node Deregister Part 5 (#89)
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* core, service_node_list: separated address from service node pubkey
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* Store service node lists for the duration of deregister lifetimes
* Quorum min/max bug, sort node list, fix node to test list
* Change quorum to store acc pub address, fix oob bug
* Code review for expiring votes, acc keys to pub_key, improve err msgs
* Add early out for is_deregistration_tx and protect against quorum changes
* Remove debug code, fix segfault
* Remove irrelevant check for tx v3 in blockchain, fix >= height for pruning quorum states
Incorrect assumption that a transaction can be kept in the chain if it could
eventually become invalid, because if it were the chain would be split and
eventually these transaction would be dropped. But also that we should not
override the pre-existing logic which handles this case anyway.
2018-07-18 04:42:47 +02:00
{
2019-10-27 23:26:38 +01:00
auto & service_node_list = m_blockchain . get_service_node_list ( ) ;
Relax deregistration rules
The replaces the deregistration mechanism with a new state change
mechanism (beginning at the v12 fork) which can change a service node's
network status via three potential values (and is extensible in the
future to handle more):
- deregistered -- this is the same as the existing deregistration; the
SN is instantly removed from the SN list.
- decommissioned -- this is a sort of temporary deregistration: your SN
remains in the service node list, but is removed from the rewards list
and from any network duties.
- recommissioned -- this tx is sent by a quorum if they observe a
decommissioned SN sending uptime proofs again. Upon reception, the SN
is reactivated and put on the end of the reward list.
Since this is broadening the quorum use, this also renames the relevant
quorum to a "obligations" quorum (since it validates SN obligations),
while the transactions are "state_change" transactions (since they
change the state of a registered SN).
The new parameters added to service_node_rules.h control how this works:
// Service node decommissioning: as service nodes stay up they earn "credits" (measured in blocks)
// towards a future outage. A new service node starts out with INITIAL_CREDIT, and then builds up
// CREDIT_PER_DAY for each day the service node remains active up to a maximum of
// DECOMMISSION_MAX_CREDIT.
//
// If a service node stops sending uptime proofs, a quorum will consider whether the service node
// has built up enough credits (at least MINIMUM): if so, instead of submitting a deregistration,
// it instead submits a decommission. This removes the service node from the list of active
// service nodes both for rewards and for any active network duties. If the service node comes
// back online (i.e. starts sending the required performance proofs again) before the credits run
// out then a quorum will reinstate the service node using a recommission transaction, which adds
// the service node back to the bottom of the service node reward list, and resets its accumulated
// credits to 0. If it does not come back online within the required number of blocks (i.e. the
// accumulated credit at the point of decommissioning) then a quorum will send a permanent
// deregistration transaction to the network, starting a 30-day deregistration count down.
This commit currently includes values (which are not necessarily
finalized):
- 8 hours (240 blocks) of credit required for activation of a
decommission (rather than a deregister)
- 0 initial credits at registration
- a maximum of 24 hours (720 blocks) of credits
- credits accumulate at a rate that you hit 24 hours of credits after 30
days of operation.
Miscellaneous other details of this PR:
- a new TX extra tag is used for the state change (including
deregistrations). The old extra tag has no version or type tag, so
couldn't be reused. The data in the new tag is slightly more
efficiently packed than the old deregistration transaction, so it gets
used for deregistrations (starting at the v12 fork) as well.
- Correct validator/worker selection required generalizing the shuffle
function to be able to shuffle just part of a vector. This lets us
stick any down service nodes at the end of the potential list, then
select validators by only shuffling the part of the index vector that
contains active service indices. Once the validators are selected, the
remainder of the list (this time including decommissioned SN indices) is
shuffled to select quorum workers to check, thus allowing decommisioned
nodes to be randomly included in the nodes to check without being
selected as a validator.
- Swarm recalculation was not quite right: swarms were recalculated on
SN registrations, even if those registrations were include shared node
registrations, but *not* recalculated on stakes. Starting with the
upgrade this behaviour is fixed (swarms aren't actually used currently
and aren't consensus-relevant so recalculating early won't hurt
anything).
- Details on decomm/dereg are added to RPC info and print_sn/print_sn_status
- Slightly improves the % of reward output in the print_sn output by
rounding it to two digits, and reserves space in the output string to
avoid excessive reallocations.
- Adds various debugging at higher debug levels to quorum voting (into
all of voting itself, vote transmission, and vote reception).
- Reset service node list internal data structure version to 0. The SN
list has to be rescanned anyway at upgrade (its size has changed), so we
might as well reset the version and remove the version-dependent
serialization code. (Note that the affected code here is for SN states
in lmdb storage, not for SN-to-SN communication serialization).
2019-06-18 23:57:02 +02:00
if ( tx . type = = txtype : : state_change )
Service Node Deregister Part 5 (#89)
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* core, service_node_list: separated address from service node pubkey
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* Store service node lists for the duration of deregister lifetimes
* Quorum min/max bug, sort node list, fix node to test list
* Change quorum to store acc pub address, fix oob bug
* Code review for expiring votes, acc keys to pub_key, improve err msgs
* Add early out for is_deregistration_tx and protect against quorum changes
* Remove debug code, fix segfault
* Remove irrelevant check for tx v3 in blockchain, fix >= height for pruning quorum states
Incorrect assumption that a transaction can be kept in the chain if it could
eventually become invalid, because if it were the chain would be split and
eventually these transaction would be dropped. But also that we should not
override the pre-existing logic which handles this case anyway.
2018-07-18 04:42:47 +02:00
{
Relax deregistration rules
The replaces the deregistration mechanism with a new state change
mechanism (beginning at the v12 fork) which can change a service node's
network status via three potential values (and is extensible in the
future to handle more):
- deregistered -- this is the same as the existing deregistration; the
SN is instantly removed from the SN list.
- decommissioned -- this is a sort of temporary deregistration: your SN
remains in the service node list, but is removed from the rewards list
and from any network duties.
- recommissioned -- this tx is sent by a quorum if they observe a
decommissioned SN sending uptime proofs again. Upon reception, the SN
is reactivated and put on the end of the reward list.
Since this is broadening the quorum use, this also renames the relevant
quorum to a "obligations" quorum (since it validates SN obligations),
while the transactions are "state_change" transactions (since they
change the state of a registered SN).
The new parameters added to service_node_rules.h control how this works:
// Service node decommissioning: as service nodes stay up they earn "credits" (measured in blocks)
// towards a future outage. A new service node starts out with INITIAL_CREDIT, and then builds up
// CREDIT_PER_DAY for each day the service node remains active up to a maximum of
// DECOMMISSION_MAX_CREDIT.
//
// If a service node stops sending uptime proofs, a quorum will consider whether the service node
// has built up enough credits (at least MINIMUM): if so, instead of submitting a deregistration,
// it instead submits a decommission. This removes the service node from the list of active
// service nodes both for rewards and for any active network duties. If the service node comes
// back online (i.e. starts sending the required performance proofs again) before the credits run
// out then a quorum will reinstate the service node using a recommission transaction, which adds
// the service node back to the bottom of the service node reward list, and resets its accumulated
// credits to 0. If it does not come back online within the required number of blocks (i.e. the
// accumulated credit at the point of decommissioning) then a quorum will send a permanent
// deregistration transaction to the network, starting a 30-day deregistration count down.
This commit currently includes values (which are not necessarily
finalized):
- 8 hours (240 blocks) of credit required for activation of a
decommission (rather than a deregister)
- 0 initial credits at registration
- a maximum of 24 hours (720 blocks) of credits
- credits accumulate at a rate that you hit 24 hours of credits after 30
days of operation.
Miscellaneous other details of this PR:
- a new TX extra tag is used for the state change (including
deregistrations). The old extra tag has no version or type tag, so
couldn't be reused. The data in the new tag is slightly more
efficiently packed than the old deregistration transaction, so it gets
used for deregistrations (starting at the v12 fork) as well.
- Correct validator/worker selection required generalizing the shuffle
function to be able to shuffle just part of a vector. This lets us
stick any down service nodes at the end of the potential list, then
select validators by only shuffling the part of the index vector that
contains active service indices. Once the validators are selected, the
remainder of the list (this time including decommissioned SN indices) is
shuffled to select quorum workers to check, thus allowing decommisioned
nodes to be randomly included in the nodes to check without being
selected as a validator.
- Swarm recalculation was not quite right: swarms were recalculated on
SN registrations, even if those registrations were include shared node
registrations, but *not* recalculated on stakes. Starting with the
upgrade this behaviour is fixed (swarms aren't actually used currently
and aren't consensus-relevant so recalculating early won't hurt
anything).
- Details on decomm/dereg are added to RPC info and print_sn/print_sn_status
- Slightly improves the % of reward output in the print_sn output by
rounding it to two digits, and reserves space in the output string to
avoid excessive reallocations.
- Adds various debugging at higher debug levels to quorum voting (into
all of voting itself, vote transmission, and vote reception).
- Reset service node list internal data structure version to 0. The SN
list has to be rescanned anyway at upgrade (its size has changed), so we
might as well reset the version and remove the version-dependent
serialization code. (Note that the affected code here is for SN states
in lmdb storage, not for SN-to-SN communication serialization).
2019-06-18 23:57:02 +02:00
tx_extra_service_node_state_change state_change ;
if ( ! get_service_node_state_change_from_tx_extra ( tx . extra , state_change , hard_fork_version ) )
Service Node Deregister Part 5 (#89)
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* core, service_node_list: separated address from service node pubkey
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* Store service node lists for the duration of deregister lifetimes
* Quorum min/max bug, sort node list, fix node to test list
* Change quorum to store acc pub address, fix oob bug
* Code review for expiring votes, acc keys to pub_key, improve err msgs
* Add early out for is_deregistration_tx and protect against quorum changes
* Remove debug code, fix segfault
* Remove irrelevant check for tx v3 in blockchain, fix >= height for pruning quorum states
Incorrect assumption that a transaction can be kept in the chain if it could
eventually become invalid, because if it were the chain would be split and
eventually these transaction would be dropped. But also that we should not
override the pre-existing logic which handles this case anyway.
2018-07-18 04:42:47 +02:00
{
2019-07-26 07:30:51 +02:00
MERROR ( " Could not get service node state change from tx: " < < get_transaction_hash ( tx ) < < " , possibly corrupt tx in your blockchain, rejecting malformed state change " ) ;
2019-07-04 09:25:02 +02:00
return false ;
}
crypto : : public_key service_node_to_change ;
2019-07-31 04:22:07 +02:00
auto const quorum_type = service_nodes : : quorum_type : : obligations ;
auto const quorum_group = service_nodes : : quorum_group : : worker ;
// NOTE: We can fail to resolve a public key if we are popping blocks greater than the number of quorums we store.
bool const can_resolve_quorum_pubkey = service_node_list . get_quorum_pubkey ( quorum_type ,
quorum_group ,
state_change . block_height ,
state_change . service_node_index ,
service_node_to_change ) ;
Service Node Deregister Part 5 (#89)
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* core, service_node_list: separated address from service node pubkey
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* Store service node lists for the duration of deregister lifetimes
* Quorum min/max bug, sort node list, fix node to test list
* Change quorum to store acc pub address, fix oob bug
* Code review for expiring votes, acc keys to pub_key, improve err msgs
* Add early out for is_deregistration_tx and protect against quorum changes
* Remove debug code, fix segfault
* Remove irrelevant check for tx v3 in blockchain, fix >= height for pruning quorum states
Incorrect assumption that a transaction can be kept in the chain if it could
eventually become invalid, because if it were the chain would be split and
eventually these transaction would be dropped. But also that we should not
override the pre-existing logic which handles this case anyway.
2018-07-18 04:42:47 +02:00
2019-01-25 04:15:52 +01:00
std : : vector < transaction > pool_txs ;
get_transactions ( pool_txs ) ;
for ( const transaction & pool_tx : pool_txs )
{
Relax deregistration rules
The replaces the deregistration mechanism with a new state change
mechanism (beginning at the v12 fork) which can change a service node's
network status via three potential values (and is extensible in the
future to handle more):
- deregistered -- this is the same as the existing deregistration; the
SN is instantly removed from the SN list.
- decommissioned -- this is a sort of temporary deregistration: your SN
remains in the service node list, but is removed from the rewards list
and from any network duties.
- recommissioned -- this tx is sent by a quorum if they observe a
decommissioned SN sending uptime proofs again. Upon reception, the SN
is reactivated and put on the end of the reward list.
Since this is broadening the quorum use, this also renames the relevant
quorum to a "obligations" quorum (since it validates SN obligations),
while the transactions are "state_change" transactions (since they
change the state of a registered SN).
The new parameters added to service_node_rules.h control how this works:
// Service node decommissioning: as service nodes stay up they earn "credits" (measured in blocks)
// towards a future outage. A new service node starts out with INITIAL_CREDIT, and then builds up
// CREDIT_PER_DAY for each day the service node remains active up to a maximum of
// DECOMMISSION_MAX_CREDIT.
//
// If a service node stops sending uptime proofs, a quorum will consider whether the service node
// has built up enough credits (at least MINIMUM): if so, instead of submitting a deregistration,
// it instead submits a decommission. This removes the service node from the list of active
// service nodes both for rewards and for any active network duties. If the service node comes
// back online (i.e. starts sending the required performance proofs again) before the credits run
// out then a quorum will reinstate the service node using a recommission transaction, which adds
// the service node back to the bottom of the service node reward list, and resets its accumulated
// credits to 0. If it does not come back online within the required number of blocks (i.e. the
// accumulated credit at the point of decommissioning) then a quorum will send a permanent
// deregistration transaction to the network, starting a 30-day deregistration count down.
This commit currently includes values (which are not necessarily
finalized):
- 8 hours (240 blocks) of credit required for activation of a
decommission (rather than a deregister)
- 0 initial credits at registration
- a maximum of 24 hours (720 blocks) of credits
- credits accumulate at a rate that you hit 24 hours of credits after 30
days of operation.
Miscellaneous other details of this PR:
- a new TX extra tag is used for the state change (including
deregistrations). The old extra tag has no version or type tag, so
couldn't be reused. The data in the new tag is slightly more
efficiently packed than the old deregistration transaction, so it gets
used for deregistrations (starting at the v12 fork) as well.
- Correct validator/worker selection required generalizing the shuffle
function to be able to shuffle just part of a vector. This lets us
stick any down service nodes at the end of the potential list, then
select validators by only shuffling the part of the index vector that
contains active service indices. Once the validators are selected, the
remainder of the list (this time including decommissioned SN indices) is
shuffled to select quorum workers to check, thus allowing decommisioned
nodes to be randomly included in the nodes to check without being
selected as a validator.
- Swarm recalculation was not quite right: swarms were recalculated on
SN registrations, even if those registrations were include shared node
registrations, but *not* recalculated on stakes. Starting with the
upgrade this behaviour is fixed (swarms aren't actually used currently
and aren't consensus-relevant so recalculating early won't hurt
anything).
- Details on decomm/dereg are added to RPC info and print_sn/print_sn_status
- Slightly improves the % of reward output in the print_sn output by
rounding it to two digits, and reserves space in the output string to
avoid excessive reallocations.
- Adds various debugging at higher debug levels to quorum voting (into
all of voting itself, vote transmission, and vote reception).
- Reset service node list internal data structure version to 0. The SN
list has to be rescanned anyway at upgrade (its size has changed), so we
might as well reset the version and remove the version-dependent
serialization code. (Note that the affected code here is for SN states
in lmdb storage, not for SN-to-SN communication serialization).
2019-06-18 23:57:02 +02:00
if ( pool_tx . type ! = txtype : : state_change )
2019-01-25 04:15:52 +01:00
continue ;
Relax deregistration rules
The replaces the deregistration mechanism with a new state change
mechanism (beginning at the v12 fork) which can change a service node's
network status via three potential values (and is extensible in the
future to handle more):
- deregistered -- this is the same as the existing deregistration; the
SN is instantly removed from the SN list.
- decommissioned -- this is a sort of temporary deregistration: your SN
remains in the service node list, but is removed from the rewards list
and from any network duties.
- recommissioned -- this tx is sent by a quorum if they observe a
decommissioned SN sending uptime proofs again. Upon reception, the SN
is reactivated and put on the end of the reward list.
Since this is broadening the quorum use, this also renames the relevant
quorum to a "obligations" quorum (since it validates SN obligations),
while the transactions are "state_change" transactions (since they
change the state of a registered SN).
The new parameters added to service_node_rules.h control how this works:
// Service node decommissioning: as service nodes stay up they earn "credits" (measured in blocks)
// towards a future outage. A new service node starts out with INITIAL_CREDIT, and then builds up
// CREDIT_PER_DAY for each day the service node remains active up to a maximum of
// DECOMMISSION_MAX_CREDIT.
//
// If a service node stops sending uptime proofs, a quorum will consider whether the service node
// has built up enough credits (at least MINIMUM): if so, instead of submitting a deregistration,
// it instead submits a decommission. This removes the service node from the list of active
// service nodes both for rewards and for any active network duties. If the service node comes
// back online (i.e. starts sending the required performance proofs again) before the credits run
// out then a quorum will reinstate the service node using a recommission transaction, which adds
// the service node back to the bottom of the service node reward list, and resets its accumulated
// credits to 0. If it does not come back online within the required number of blocks (i.e. the
// accumulated credit at the point of decommissioning) then a quorum will send a permanent
// deregistration transaction to the network, starting a 30-day deregistration count down.
This commit currently includes values (which are not necessarily
finalized):
- 8 hours (240 blocks) of credit required for activation of a
decommission (rather than a deregister)
- 0 initial credits at registration
- a maximum of 24 hours (720 blocks) of credits
- credits accumulate at a rate that you hit 24 hours of credits after 30
days of operation.
Miscellaneous other details of this PR:
- a new TX extra tag is used for the state change (including
deregistrations). The old extra tag has no version or type tag, so
couldn't be reused. The data in the new tag is slightly more
efficiently packed than the old deregistration transaction, so it gets
used for deregistrations (starting at the v12 fork) as well.
- Correct validator/worker selection required generalizing the shuffle
function to be able to shuffle just part of a vector. This lets us
stick any down service nodes at the end of the potential list, then
select validators by only shuffling the part of the index vector that
contains active service indices. Once the validators are selected, the
remainder of the list (this time including decommissioned SN indices) is
shuffled to select quorum workers to check, thus allowing decommisioned
nodes to be randomly included in the nodes to check without being
selected as a validator.
- Swarm recalculation was not quite right: swarms were recalculated on
SN registrations, even if those registrations were include shared node
registrations, but *not* recalculated on stakes. Starting with the
upgrade this behaviour is fixed (swarms aren't actually used currently
and aren't consensus-relevant so recalculating early won't hurt
anything).
- Details on decomm/dereg are added to RPC info and print_sn/print_sn_status
- Slightly improves the % of reward output in the print_sn output by
rounding it to two digits, and reserves space in the output string to
avoid excessive reallocations.
- Adds various debugging at higher debug levels to quorum voting (into
all of voting itself, vote transmission, and vote reception).
- Reset service node list internal data structure version to 0. The SN
list has to be rescanned anyway at upgrade (its size has changed), so we
might as well reset the version and remove the version-dependent
serialization code. (Note that the affected code here is for SN states
in lmdb storage, not for SN-to-SN communication serialization).
2019-06-18 23:57:02 +02:00
tx_extra_service_node_state_change pool_tx_state_change ;
if ( ! get_service_node_state_change_from_tx_extra ( pool_tx . extra , pool_tx_state_change , hard_fork_version ) )
2019-01-25 04:15:52 +01:00
{
2019-10-15 09:14:47 +02:00
LOG_PRINT_L1 ( " Could not get service node state change from tx: " < < get_transaction_hash ( pool_tx ) < < " , possibly corrupt tx in the pool " ) ;
2019-01-25 04:15:52 +01:00
continue ;
}
2019-07-09 05:59:42 +02:00
if ( hard_fork_version > = cryptonote : : network_version_12_checkpointing )
2019-07-04 09:25:02 +02:00
{
2019-07-09 05:59:42 +02:00
crypto : : public_key service_node_to_change_in_the_pool ;
2019-07-31 04:22:07 +02:00
bool same_service_node = false ;
if ( can_resolve_quorum_pubkey & & service_node_list . get_quorum_pubkey ( quorum_type , quorum_group , pool_tx_state_change . block_height , pool_tx_state_change . service_node_index , service_node_to_change_in_the_pool ) )
2019-07-09 05:59:42 +02:00
{
2019-07-31 04:22:07 +02:00
same_service_node = ( service_node_to_change = = service_node_to_change_in_the_pool ) ;
2019-07-09 05:59:42 +02:00
}
else
{
2019-07-31 04:22:07 +02:00
same_service_node = ( state_change = = pool_tx_state_change ) ;
2019-07-09 05:59:42 +02:00
}
2019-07-31 04:22:07 +02:00
if ( same_service_node & & pool_tx_state_change . state = = state_change . state )
2019-07-04 09:25:02 +02:00
return true ;
}
else
{
if ( state_change = = pool_tx_state_change )
return true ;
}
2019-01-25 04:15:52 +01:00
}
}
2019-06-11 20:53:46 +02:00
else if ( tx . type = = txtype : : key_image_unlock )
2019-01-25 04:15:52 +01:00
{
tx_extra_tx_key_image_unlock unlock ;
2020-06-02 06:21:05 +02:00
if ( ! cryptonote : : get_field_from_tx_extra ( tx . extra , unlock ) )
Service Node Deregister Part 5 (#89)
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* core, service_node_list: separated address from service node pubkey
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* Store service node lists for the duration of deregister lifetimes
* Quorum min/max bug, sort node list, fix node to test list
* Change quorum to store acc pub address, fix oob bug
* Code review for expiring votes, acc keys to pub_key, improve err msgs
* Add early out for is_deregistration_tx and protect against quorum changes
* Remove debug code, fix segfault
* Remove irrelevant check for tx v3 in blockchain, fix >= height for pruning quorum states
Incorrect assumption that a transaction can be kept in the chain if it could
eventually become invalid, because if it were the chain would be split and
eventually these transaction would be dropped. But also that we should not
override the pre-existing logic which handles this case anyway.
2018-07-18 04:42:47 +02:00
{
2019-07-26 07:30:51 +02:00
MERROR ( " Could not get key image unlock from tx: " < < get_transaction_hash ( tx ) < < " , tx to add is possibly invalid, rejecting " ) ;
Service Node Deregister Part 5 (#89)
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* core, service_node_list: separated address from service node pubkey
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* Store service node lists for the duration of deregister lifetimes
* Quorum min/max bug, sort node list, fix node to test list
* Change quorum to store acc pub address, fix oob bug
* Code review for expiring votes, acc keys to pub_key, improve err msgs
* Add early out for is_deregistration_tx and protect against quorum changes
* Remove debug code, fix segfault
* Remove irrelevant check for tx v3 in blockchain, fix >= height for pruning quorum states
Incorrect assumption that a transaction can be kept in the chain if it could
eventually become invalid, because if it were the chain would be split and
eventually these transaction would be dropped. But also that we should not
override the pre-existing logic which handles this case anyway.
2018-07-18 04:42:47 +02:00
return true ;
}
2019-01-25 04:15:52 +01:00
std : : vector < transaction > pool_txs ;
get_transactions ( pool_txs ) ;
for ( const transaction & pool_tx : pool_txs )
{
2019-06-11 20:53:46 +02:00
if ( pool_tx . type ! = tx . type )
2019-01-25 04:15:52 +01:00
continue ;
tx_extra_tx_key_image_unlock pool_unlock ;
2020-06-02 06:21:05 +02:00
if ( ! cryptonote : : get_field_from_tx_extra ( pool_tx . extra , pool_unlock ) )
2019-01-25 04:15:52 +01:00
{
2019-10-15 09:14:47 +02:00
LOG_PRINT_L1 ( " Could not get key image unlock from tx: " < < get_transaction_hash ( tx ) < < " , possibly corrupt tx in the pool " ) ;
2019-01-25 04:15:52 +01:00
return true ;
}
Relax deregistration rules
The replaces the deregistration mechanism with a new state change
mechanism (beginning at the v12 fork) which can change a service node's
network status via three potential values (and is extensible in the
future to handle more):
- deregistered -- this is the same as the existing deregistration; the
SN is instantly removed from the SN list.
- decommissioned -- this is a sort of temporary deregistration: your SN
remains in the service node list, but is removed from the rewards list
and from any network duties.
- recommissioned -- this tx is sent by a quorum if they observe a
decommissioned SN sending uptime proofs again. Upon reception, the SN
is reactivated and put on the end of the reward list.
Since this is broadening the quorum use, this also renames the relevant
quorum to a "obligations" quorum (since it validates SN obligations),
while the transactions are "state_change" transactions (since they
change the state of a registered SN).
The new parameters added to service_node_rules.h control how this works:
// Service node decommissioning: as service nodes stay up they earn "credits" (measured in blocks)
// towards a future outage. A new service node starts out with INITIAL_CREDIT, and then builds up
// CREDIT_PER_DAY for each day the service node remains active up to a maximum of
// DECOMMISSION_MAX_CREDIT.
//
// If a service node stops sending uptime proofs, a quorum will consider whether the service node
// has built up enough credits (at least MINIMUM): if so, instead of submitting a deregistration,
// it instead submits a decommission. This removes the service node from the list of active
// service nodes both for rewards and for any active network duties. If the service node comes
// back online (i.e. starts sending the required performance proofs again) before the credits run
// out then a quorum will reinstate the service node using a recommission transaction, which adds
// the service node back to the bottom of the service node reward list, and resets its accumulated
// credits to 0. If it does not come back online within the required number of blocks (i.e. the
// accumulated credit at the point of decommissioning) then a quorum will send a permanent
// deregistration transaction to the network, starting a 30-day deregistration count down.
This commit currently includes values (which are not necessarily
finalized):
- 8 hours (240 blocks) of credit required for activation of a
decommission (rather than a deregister)
- 0 initial credits at registration
- a maximum of 24 hours (720 blocks) of credits
- credits accumulate at a rate that you hit 24 hours of credits after 30
days of operation.
Miscellaneous other details of this PR:
- a new TX extra tag is used for the state change (including
deregistrations). The old extra tag has no version or type tag, so
couldn't be reused. The data in the new tag is slightly more
efficiently packed than the old deregistration transaction, so it gets
used for deregistrations (starting at the v12 fork) as well.
- Correct validator/worker selection required generalizing the shuffle
function to be able to shuffle just part of a vector. This lets us
stick any down service nodes at the end of the potential list, then
select validators by only shuffling the part of the index vector that
contains active service indices. Once the validators are selected, the
remainder of the list (this time including decommissioned SN indices) is
shuffled to select quorum workers to check, thus allowing decommisioned
nodes to be randomly included in the nodes to check without being
selected as a validator.
- Swarm recalculation was not quite right: swarms were recalculated on
SN registrations, even if those registrations were include shared node
registrations, but *not* recalculated on stakes. Starting with the
upgrade this behaviour is fixed (swarms aren't actually used currently
and aren't consensus-relevant so recalculating early won't hurt
anything).
- Details on decomm/dereg are added to RPC info and print_sn/print_sn_status
- Slightly improves the % of reward output in the print_sn output by
rounding it to two digits, and reserves space in the output string to
avoid excessive reallocations.
- Adds various debugging at higher debug levels to quorum voting (into
all of voting itself, vote transmission, and vote reception).
- Reset service node list internal data structure version to 0. The SN
list has to be rescanned anyway at upgrade (its size has changed), so we
might as well reset the version and remove the version-dependent
serialization code. (Note that the affected code here is for SN states
in lmdb storage, not for SN-to-SN communication serialization).
2019-06-18 23:57:02 +02:00
if ( unlock = = pool_unlock )
2019-01-25 04:15:52 +01:00
{
2019-07-26 07:30:51 +02:00
LOG_PRINT_L1 ( " New TX: " < < get_transaction_hash ( tx ) < < " , has TX: " < < get_transaction_hash ( pool_tx ) < < " from the pool that is requesting to unlock the same key image already. " ) ;
2019-01-25 04:15:52 +01:00
return true ;
}
}
}
2021-01-04 01:09:45 +01:00
else if ( tx . type = = txtype : : oxen_name_system )
2019-11-01 01:58:48 +01:00
{
2021-01-04 01:09:45 +01:00
tx_extra_oxen_name_system data ;
2020-06-02 06:21:05 +02:00
if ( ! cryptonote : : get_field_from_tx_extra ( tx . extra , data ) )
2019-11-01 01:58:48 +01:00
{
MERROR ( " Could not get acquire name service from tx: " < < get_transaction_hash ( tx ) < < " , tx to add is possibly invalid, rejecting " ) ;
return true ;
}
std : : vector < transaction > pool_txs ;
get_transactions ( pool_txs ) ;
for ( const transaction & pool_tx : pool_txs )
{
if ( pool_tx . type ! = tx . type )
continue ;
2021-01-04 01:09:45 +01:00
tx_extra_oxen_name_system pool_data ;
2020-06-02 06:21:05 +02:00
if ( ! cryptonote : : get_field_from_tx_extra ( pool_tx . extra , pool_data ) )
2019-11-01 01:58:48 +01:00
{
LOG_PRINT_L1 ( " Could not get acquire name service from tx: " < < get_transaction_hash ( tx ) < < " , possibly corrupt tx in the pool " ) ;
return true ;
}
2020-02-10 06:50:59 +01:00
if ( data . type = = pool_data . type & & data . name_hash = = pool_data . name_hash )
2019-11-01 01:58:48 +01:00
{
2021-02-12 05:19:30 +01:00
LOG_PRINT_L1 ( " New TX: " < < get_transaction_hash ( tx ) < < " , has TX: " < < get_transaction_hash ( pool_tx ) < < " from the pool that is requesting the same ONS entry already. " ) ;
2019-11-01 01:58:48 +01:00
return true ;
}
}
}
2019-01-25 04:15:52 +01:00
else
{
2019-11-01 01:58:48 +01:00
if ( tx . type ! = txtype : : standard & & tx . type ! = txtype : : stake )
{
2021-01-04 01:09:45 +01:00
// NOTE(oxen): This is a developer error. If we come across this in production, be conservative and just reject
2019-11-01 01:58:48 +01:00
MERROR ( " Unrecognised transaction type: " < < tx . type < < " for tx: " < < get_transaction_hash ( tx ) ) ;
return true ;
}
Service Node Deregister Part 5 (#89)
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* core, service_node_list: separated address from service node pubkey
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* Store service node lists for the duration of deregister lifetimes
* Quorum min/max bug, sort node list, fix node to test list
* Change quorum to store acc pub address, fix oob bug
* Code review for expiring votes, acc keys to pub_key, improve err msgs
* Add early out for is_deregistration_tx and protect against quorum changes
* Remove debug code, fix segfault
* Remove irrelevant check for tx v3 in blockchain, fix >= height for pruning quorum states
Incorrect assumption that a transaction can be kept in the chain if it could
eventually become invalid, because if it were the chain would be split and
eventually these transaction would be dropped. But also that we should not
override the pre-existing logic which handles this case anyway.
2018-07-18 04:42:47 +02:00
}
return false ;
2014-03-03 23:07:58 +01:00
}
2019-10-27 23:47:19 +01:00
// Blink notes: a blink quorum member adds an incoming blink tx into the mempool to make sure it
// can be accepted, but sets it as do_not_relay initially. If it gets added, the quorum member
// sends a signature to other quorum members. Once enough signatures are received it updates it
// to set `do_not_relay` to false and starts relaying it (other quorum members do the same).
2014-03-03 23:07:58 +01:00
//---------------------------------------------------------------------------------
2019-11-25 05:57:15 +01:00
bool tx_memory_pool : : add_tx ( transaction & tx , const crypto : : hash & id , const cryptonote : : blobdata & blob , size_t tx_weight , tx_verification_context & tvc , const tx_pool_options & opts , uint8_t hf_version ,
uint64_t * blink_rollback_height )
2014-03-03 23:07:58 +01:00
{
2017-05-31 23:50:12 +02:00
// this should already be called with that lock, but let's make it explicit for clarity
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
std : : unique_lock lock { m_transactions_lock } ;
2017-05-31 23:50:12 +02:00
2016-10-10 21:47:52 +02:00
PERF_TIMER ( add_tx ) ;
2019-06-11 20:53:46 +02:00
if ( tx . version = = txversion : : v0 )
2016-06-16 00:37:13 +02:00
{
// v0 never accepted
2016-07-27 11:10:38 +02:00
LOG_PRINT_L1 ( " transaction version 0 is invalid " ) ;
2016-06-16 00:37:13 +02:00
tvc . m_verifivation_failed = true ;
return false ;
}
2016-06-30 00:00:20 +02:00
2016-01-29 18:15:09 +01:00
// we do not accept transactions that timed out before, unless they're
// kept_by_block
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
if ( ! opts . kept_by_block & & m_timed_out_transactions . find ( id ) ! = m_timed_out_transactions . end ( ) )
2016-01-29 18:15:09 +01:00
{
// not clear if we should set that, since verifivation (sic) did not fail before, since
// the tx was accepted before timing out.
tvc . m_verifivation_failed = true ;
return false ;
}
2014-03-03 23:07:58 +01:00
2022-04-29 01:51:14 +02:00
if ( ! check_inputs_types_supported ( tx ) )
2014-03-03 23:07:58 +01:00
{
tvc . m_verifivation_failed = true ;
2016-03-27 13:35:36 +02:00
tvc . m_invalid_input = true ;
2014-03-03 23:07:58 +01:00
return false ;
}
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
uint64_t fee , burned ;
2016-06-16 00:37:13 +02:00
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
if ( ! get_tx_miner_fee ( tx , fee , hf_version > = HF_VERSION_FEE_BURNING , & burned ) )
2014-03-03 23:07:58 +01:00
{
2019-11-25 05:57:15 +01:00
// This code is a bit convoluted: the above sets `fee`, and returns false for a pre-ringct tx
// with a too-low fee, but for ringct (v2+) txes it just sets `fee` but doesn't check it and
// always returns true: the actual v2 tx fee amount gets tested in the check_fee call below
2019-10-30 22:28:47 +01:00
tvc . m_verifivation_failed = true ;
tvc . m_fee_too_low = true ;
2019-11-25 05:57:15 +01:00
return false ;
2014-03-03 23:07:58 +01:00
}
2022-04-29 01:51:14 +02:00
if ( hf_version < cryptonote : : network_version_19 )
2014-07-17 17:05:46 +02:00
{
2022-04-29 01:51:14 +02:00
if ( ! opts . kept_by_block & & tx . is_transfer ( ) & & ! m_blockchain . check_fee ( tx_weight , tx . vout . size ( ) , fee , burned , opts ) )
{
tvc . m_verifivation_failed = true ;
tvc . m_fee_too_low = true ;
return false ;
}
2014-07-17 17:05:46 +02:00
}
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
size_t tx_weight_limit = get_transaction_weight_limit ( hf_version ) ;
if ( ( ! opts . kept_by_block | | hf_version > = HF_VERSION_PER_BYTE_FEE ) & & tx_weight > tx_weight_limit )
2014-07-17 17:02:05 +02:00
{
2018-07-18 23:24:53 +02:00
LOG_PRINT_L1 ( " transaction is too heavy: " < < tx_weight < < " bytes, maximum weight: " < < tx_weight_limit ) ;
2014-03-03 23:07:58 +01:00
tvc . m_verifivation_failed = true ;
2016-03-27 13:35:36 +02:00
tvc . m_too_big = true ;
2014-03-03 23:07:58 +01:00
return false ;
}
{
2019-11-25 05:57:15 +01:00
std : : vector < crypto : : hash > conflict_txs ;
2019-12-25 05:11:08 +01:00
bool double_spend = have_tx_keyimges_as_spent ( tx , & conflict_txs ) ;
2019-11-25 05:57:15 +01:00
if ( double_spend )
2014-03-03 23:07:58 +01:00
{
2019-11-25 05:57:15 +01:00
if ( opts . kept_by_block )
{
// The tx came from a block popped from the chain; we keep it around even if the key
// images are spent so that we notice the double spend *unless* the tx is conflicting with
// one or more blink txs, in which case we drop it because it can never be accepted.
2019-12-25 05:11:08 +01:00
auto blink_lock = blink_shared_lock ( ) ;
2019-11-25 05:57:15 +01:00
double_spend = false ;
for ( const auto & tx_hash : conflict_txs )
{
2019-12-26 07:38:00 +01:00
if ( tx_hash ! = id & & m_blinks . count ( tx_hash ) )
2019-11-25 05:57:15 +01:00
{
2019-12-26 07:38:00 +01:00
// Warn on this because it almost certainly indicates something malicious
MWARNING ( " Not re-adding popped/incoming tx " < < id < < " to the mempool: it conflicts with blink tx " < < tx_hash ) ;
2019-11-25 05:57:15 +01:00
double_spend = true ;
break ;
}
}
}
else if ( opts . approved_blink )
{
2019-12-25 05:16:29 +01:00
MDEBUG ( " Incoming blink tx is approved, but has " < < conflict_txs . size ( ) < < " conflicting local tx(es); dropping conflicts " ) ;
2019-11-25 05:57:15 +01:00
if ( remove_blink_conflicts ( id , conflict_txs , blink_rollback_height ) )
double_spend = false ;
else
MERROR ( " Blink error: incoming blink tx cannot be accepted as it conflicts with checkpointed txs " ) ;
}
if ( double_spend )
{
mark_double_spend ( tx ) ;
LOG_PRINT_L1 ( " Transaction with id= " < < id < < " used already spent key images " ) ;
tvc . m_verifivation_failed = true ;
tvc . m_double_spend = true ;
return false ;
}
2014-03-03 23:07:58 +01:00
}
2019-11-30 17:13:40 +01:00
}
if ( ! opts . kept_by_block & & have_duplicated_non_standard_tx ( tx , hf_version ) )
{
mark_double_spend ( tx ) ;
LOG_PRINT_L1 ( " Transaction with id= " < < id < < " already has a duplicate tx for height " ) ;
tvc . m_verifivation_failed = true ;
tvc . m_double_spend = true ;
return false ;
Service Node Deregister Part 5 (#89)
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* core, service_node_list: separated address from service node pubkey
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* Store service node lists for the duration of deregister lifetimes
* Quorum min/max bug, sort node list, fix node to test list
* Change quorum to store acc pub address, fix oob bug
* Code review for expiring votes, acc keys to pub_key, improve err msgs
* Add early out for is_deregistration_tx and protect against quorum changes
* Remove debug code, fix segfault
* Remove irrelevant check for tx v3 in blockchain, fix >= height for pruning quorum states
Incorrect assumption that a transaction can be kept in the chain if it could
eventually become invalid, because if it were the chain would be split and
eventually these transaction would be dropped. But also that we should not
override the pre-existing logic which handles this case anyway.
2018-07-18 04:42:47 +02:00
}
2016-03-27 13:35:36 +02:00
if ( ! m_blockchain . check_tx_outputs ( tx , tvc ) )
2015-10-06 17:22:19 +02:00
{
2016-06-16 00:37:13 +02:00
LOG_PRINT_L1 ( " Transaction with id= " < < id < < " has at least one invalid output " ) ;
2015-10-06 17:22:19 +02:00
tvc . m_verifivation_failed = true ;
2016-03-27 13:35:36 +02:00
tvc . m_invalid_output = true ;
2015-10-06 17:22:19 +02:00
return false ;
}
2014-03-03 23:07:58 +01:00
2017-09-10 13:42:36 +02:00
// assume failure during verification steps until success is certain
tvc . m_verifivation_failed = true ;
2017-01-13 16:08:37 +01:00
time_t receive_time = time ( nullptr ) ;
2014-03-03 23:07:58 +01:00
crypto : : hash max_used_block_id = null_hash ;
uint64_t max_used_block_height = 0 ;
2017-05-14 15:06:55 +02:00
cryptonote : : txpool_tx_meta_t meta ;
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
bool inputs_okay = check_tx_inputs ( [ & tx ] ( ) - > cryptonote : : transaction & { return tx ; } , id , max_used_block_height , max_used_block_id , tvc , opts . kept_by_block ,
2019-12-26 07:38:00 +01:00
opts . approved_blink ? blink_rollback_height : nullptr ) ;
2019-11-29 04:36:43 +01:00
const bool non_standard_tx = ! tx . is_transfer ( ) ;
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
if ( ! inputs_okay )
2014-03-03 23:07:58 +01:00
{
2016-03-25 01:03:02 +01:00
// if the transaction was valid before (kept_by_block), then it
// may become valid again, so ignore the failed inputs check.
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
if ( opts . kept_by_block )
2014-03-03 23:07:58 +01:00
{
2018-07-18 23:24:53 +02:00
meta . weight = tx_weight ;
2017-05-14 15:06:55 +02:00
meta . fee = fee ;
meta . max_used_block_id = null_hash ;
meta . max_used_block_height = 0 ;
meta . last_failed_height = 0 ;
meta . last_failed_id = null_hash ;
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
meta . kept_by_block = opts . kept_by_block ;
2017-05-14 15:06:55 +02:00
meta . receive_time = receive_time ;
meta . last_relayed_time = time ( NULL ) ;
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
meta . relayed = opts . relayed ;
meta . do_not_relay = opts . do_not_relay ;
meta . double_spend_seen = ( have_tx_keyimges_as_spent ( tx ) | | have_duplicated_non_standard_tx ( tx , hf_version ) ) ;
2018-06-05 20:59:45 +02:00
meta . bf_padding = 0 ;
2017-07-02 19:43:23 +02:00
memset ( meta . padding , 0 , sizeof ( meta . padding ) ) ;
2017-05-14 15:06:55 +02:00
try
{
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
m_parsed_tx_cache . insert ( std : : make_pair ( id , tx ) ) ;
std : : unique_lock b_lock { m_blockchain } ;
2017-05-14 15:06:55 +02:00
LockedTXN lock ( m_blockchain ) ;
2018-10-31 09:31:13 +01:00
m_blockchain . add_txpool_tx ( id , blob , meta ) ;
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
if ( ! insert_key_images ( tx , id , opts . kept_by_block ) )
2017-05-14 15:06:55 +02:00
return false ;
2020-05-25 03:48:32 +02:00
m_txs_by_fee_and_receive_time . emplace ( std : : tuple < bool , double , std : : time_t > ( non_standard_tx , fee / ( double ) ( tx_weight ? tx_weight : 1 ) , receive_time ) , id ) ;
2019-06-26 06:00:05 +02:00
lock . commit ( ) ;
2017-05-14 15:06:55 +02:00
}
catch ( const std : : exception & e )
{
2019-10-29 11:43:31 +01:00
MERROR ( " Error adding transaction to txpool: " < < e . what ( ) ) ;
2017-05-14 15:06:55 +02:00
return false ;
}
2014-03-03 23:07:58 +01:00
tvc . m_verifivation_impossible = true ;
tvc . m_added_to_pool = true ;
} else
{
2014-09-09 11:32:00 +02:00
LOG_PRINT_L1 ( " tx used wrong inputs, rejected " ) ;
2014-03-03 23:07:58 +01:00
tvc . m_verifivation_failed = true ;
2017-09-16 12:27:26 +02:00
tvc . m_invalid_input = true ;
2014-03-03 23:07:58 +01:00
return false ;
}
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
}
else
2014-03-03 23:07:58 +01:00
{
//update transactions container
2018-07-18 23:24:53 +02:00
meta . weight = tx_weight ;
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
meta . kept_by_block = opts . kept_by_block ;
2017-05-14 15:06:55 +02:00
meta . fee = fee ;
meta . max_used_block_id = max_used_block_id ;
meta . max_used_block_height = max_used_block_height ;
meta . last_failed_height = 0 ;
meta . last_failed_id = null_hash ;
meta . receive_time = receive_time ;
meta . last_relayed_time = time ( NULL ) ;
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
meta . relayed = opts . relayed ;
meta . do_not_relay = opts . do_not_relay ;
2017-09-22 14:57:20 +02:00
meta . double_spend_seen = false ;
2018-06-05 20:59:45 +02:00
meta . bf_padding = 0 ;
2017-07-02 19:43:23 +02:00
memset ( meta . padding , 0 , sizeof ( meta . padding ) ) ;
2017-05-14 15:06:55 +02:00
try
{
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
if ( opts . kept_by_block )
2018-10-31 14:52:16 +01:00
m_parsed_tx_cache . insert ( std : : make_pair ( id , tx ) ) ;
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
std : : unique_lock b_lock { m_blockchain } ;
2017-05-14 15:06:55 +02:00
LockedTXN lock ( m_blockchain ) ;
2018-10-31 14:51:48 +01:00
m_blockchain . remove_txpool_tx ( id ) ;
m_blockchain . add_txpool_tx ( id , blob , meta ) ;
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
if ( ! insert_key_images ( tx , id , opts . kept_by_block ) )
2017-05-14 15:06:55 +02:00
return false ;
2020-05-25 03:48:32 +02:00
m_txs_by_fee_and_receive_time . emplace ( std : : tuple < bool , double , std : : time_t > ( non_standard_tx , fee / ( double ) ( tx_weight ? tx_weight : 1 ) , receive_time ) , id ) ;
2019-06-26 06:00:05 +02:00
lock . commit ( ) ;
2017-05-14 15:06:55 +02:00
}
catch ( const std : : exception & e )
{
2019-10-29 11:43:31 +01:00
MERROR ( " internal error: error adding transaction to txpool: " < < e . what ( ) ) ;
2017-05-14 15:06:55 +02:00
return false ;
}
2014-03-03 23:07:58 +01:00
tvc . m_added_to_pool = true ;
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
if ( ( meta . fee > 0 | | non_standard_tx ) & & ! opts . do_not_relay )
2014-03-03 23:07:58 +01:00
tvc . m_should_be_relayed = true ;
}
tvc . m_verifivation_failed = false ;
2018-07-18 23:24:53 +02:00
m_txpool_weight + = tx_weight ;
2015-04-30 07:02:12 +02:00
2016-04-17 12:04:01 +02:00
+ + m_cookie ;
2019-10-25 18:56:55 +02:00
MINFO ( " Transaction added to pool: txid " < < id < < " weight: " < < tx_weight < < " fee/byte: " < < ( fee / ( double ) ( tx_weight ? tx_weight : 1 ) ) ) ;
2018-01-29 22:20:24 +01:00
2020-05-11 20:51:01 +02:00
if ( ! opts . kept_by_block & & ! opts . do_not_relay )
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
for ( auto & notify : m_tx_notify )
notify ( id , tx , blob , opts ) ;
2019-11-25 05:57:15 +01:00
prune ( id ) ;
2018-01-29 22:20:24 +01:00
2014-03-03 23:07:58 +01:00
return true ;
}
//---------------------------------------------------------------------------------
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
bool tx_memory_pool : : add_tx ( transaction & tx , tx_verification_context & tvc , const tx_pool_options & opts , uint8_t version )
2014-03-03 23:07:58 +01:00
{
crypto : : hash h = null_hash ;
2014-09-25 08:24:42 +02:00
size_t blob_size = 0 ;
2018-10-31 09:31:13 +01:00
cryptonote : : blobdata bl ;
t_serializable_object_to_blob ( tx , bl ) ;
if ( bl . size ( ) = = 0 | | ! get_transaction_hash ( tx , h ) )
2017-09-10 13:26:24 +02:00
return false ;
Generic burn fee checking + blink burn fee checking
This adds the ability for check_fee() to also check the burn amount.
This requires passing extra info through `add_tx()` (and the various
things that call it), so I took the:
bool keeped_by_block, bool relayed, bool do_not_relay
argument triplet, moved it into a struct in tx_pool.h, then added the other fee
options there (along with some static factory functions for generating the
typical sets of option).
The majority of this commit is chasing that change through the codebase and
test suite.
This is used by blink but should also help LNS and other future burn
transactions to verify a burn amount simply when adding the transation to the
mempool. It supports a fixed burn amount, a burn amount as a multiple of the
minimum tx fee, and also allows you to increase the minimum tx fee (so that,
for example, we could require blink txes to pay miners 250% of the usual
minimum (unimportant) priority tx fee.
- Removed a useless core::add_new_tx() overload that wasn't used anywhere.
Blink-specific changes:
(I'd normally separate these into a separate commit, but they got interwoven
fairly heavily with the above change).
- changed the way blink burning is specified so that we have three knobs for
fee adjustment (fixed burn fee; base fee multiple; and required miner tx fee).
The fixed amount is currently 0, base fee is 400%, and require miner tx fee is
simply 100% (i.e. no different than a normal transaction). This is the same as
before this commit, but is changing how they are being specified in
cryptonote_config.h.
- blink tx fee, burn amount, and miner tx fee (if > 100%) now get checked
before signing a blink tx. (These fee checks don't apply to anyone else --
when propagating over the network only the miner tx fee is checked).
- Added a couple of checks for blink quorums: 1) make sure they have reached
the blink hf; 2) make sure the submitted tx version conforms to the current hf
min/max tx version.
- print blink fee information in simplewallet's `fee` output
- add "typical" fee calculations in the `fee` output:
[wallet T6SCwL (has locked stakes)]: fee
Current fee is 0.000000850 loki per byte + 0.020000000 loki per output
No backlog at priority 1
No backlog at priority 2
No backlog at priority 3
No backlog at priority 4
Current blink fee is 0.000004250 loki per byte + 0.100000000 loki per output
Estimated typical small transaction fees: 0.042125000 (unimportant), 0.210625000 (normal), 1.053125000 (elevated), 5.265625000 (priority), 0.210625000 (blink)
where "small" here is the same tx size (2500 bytes + 2 outputs) used to
estimate backlogs.
2019-11-09 04:14:15 +01:00
return add_tx ( tx , h , bl , get_transaction_weight ( tx , bl . size ( ) ) , tvc , opts , version ) ;
2014-03-03 23:07:58 +01:00
}
//---------------------------------------------------------------------------------
2019-11-06 07:28:33 +01:00
bool tx_memory_pool : : add_new_blink ( const std : : shared_ptr < blink_tx > & blink_ptr , tx_verification_context & tvc , bool & blink_exists )
2019-10-27 23:47:19 +01:00
{
assert ( ( bool ) blink_ptr ) ;
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
std : : unique_lock lock { m_transactions_lock } ;
2019-11-06 07:28:33 +01:00
auto & blink = * blink_ptr ;
2020-10-16 00:44:36 +02:00
auto & tx = var : : get < transaction > ( blink . tx ) ; // will throw if just a hash w/o a transaction
2019-10-27 23:47:19 +01:00
auto txhash = get_transaction_hash ( tx ) ;
2019-11-06 07:28:33 +01:00
2019-11-24 20:02:35 +01:00
{
auto lock = blink_shared_lock ( ) ;
blink_exists = m_blinks . count ( txhash ) ;
if ( blink_exists )
return false ;
}
2019-11-06 07:28:33 +01:00
bool approved = blink . approved ( ) ;
2021-06-18 01:06:51 +02:00
auto hf_version = m_blockchain . get_network_version ( blink . height ) ;
2020-02-18 22:47:37 +01:00
bool result = add_tx ( tx , tvc , tx_pool_options : : new_blink ( approved , hf_version ) , hf_version ) ;
2019-11-06 07:28:33 +01:00
if ( result & & approved )
2019-11-24 20:02:35 +01:00
{
auto lock = blink_unique_lock ( ) ;
2019-11-06 07:28:33 +01:00
m_blinks [ txhash ] = blink_ptr ;
2019-11-24 20:02:35 +01:00
}
else if ( ! result )
{
// Adding failed, but might have failed because another thread inserted it, so check again for
// existence of the blink
auto lock = blink_shared_lock ( ) ;
blink_exists = m_blinks . count ( txhash ) ;
}
2019-11-06 07:28:33 +01:00
return result ;
}
//---------------------------------------------------------------------------------
Fix rare blink pool deadlock
cryptonote_protocol_handler calls `pool.get_blink(hash)` while already
holding a blink shared lock, which should have been
`pool.get_blink(hash, true)` to avoid `get_blink` trying to take its own
lock.
That double lock is undefined behaviour and can cause a deadlock on the
mutex, although it appears rare that it actually does. If it does,
however, this eventually backs up into vote relaying during the idle
loop, which then stalls the idle loop so we stop sending out uptime
proofs (since that is also in the idle loop).
A simple fix here is to add the `true` argument, but on reconsideration
this extra argument to take or not take a lock is messy and error prone,
so this commit instead removes the second argument entirely and instead
documents which call must and must not hold a lock, getting rid of the
three methods (get_blink, has_blink, and add_existing_blink) that had
the `have_lock` argument. This ends up having only a small impact on
calling code - the vast majority of callers already hold a lock, and the
few that don't are easily adjusted.
2020-01-18 19:01:45 +01:00
bool tx_memory_pool : : add_existing_blink ( std : : shared_ptr < blink_tx > blink_ptr )
2019-11-06 07:28:33 +01:00
{
assert ( blink_ptr & & blink_ptr - > approved ( ) ) ;
auto & ptr = m_blinks [ blink_ptr - > get_txhash ( ) ] ;
if ( ptr )
return false ;
ptr = blink_ptr ;
return true ;
2019-10-27 23:47:19 +01:00
}
//---------------------------------------------------------------------------------
Fix rare blink pool deadlock
cryptonote_protocol_handler calls `pool.get_blink(hash)` while already
holding a blink shared lock, which should have been
`pool.get_blink(hash, true)` to avoid `get_blink` trying to take its own
lock.
That double lock is undefined behaviour and can cause a deadlock on the
mutex, although it appears rare that it actually does. If it does,
however, this eventually backs up into vote relaying during the idle
loop, which then stalls the idle loop so we stop sending out uptime
proofs (since that is also in the idle loop).
A simple fix here is to add the `true` argument, but on reconsideration
this extra argument to take or not take a lock is messy and error prone,
so this commit instead removes the second argument entirely and instead
documents which call must and must not hold a lock, getting rid of the
three methods (get_blink, has_blink, and add_existing_blink) that had
the `have_lock` argument. This ends up having only a small impact on
calling code - the vast majority of callers already hold a lock, and the
few that don't are easily adjusted.
2020-01-18 19:01:45 +01:00
std : : shared_ptr < blink_tx > tx_memory_pool : : get_blink ( const crypto : : hash & tx_hash ) const
2019-10-27 23:47:19 +01:00
{
auto it = m_blinks . find ( tx_hash ) ;
if ( it ! = m_blinks . end ( ) )
return it - > second ;
return { } ;
}
//---------------------------------------------------------------------------------
Fix rare blink pool deadlock
cryptonote_protocol_handler calls `pool.get_blink(hash)` while already
holding a blink shared lock, which should have been
`pool.get_blink(hash, true)` to avoid `get_blink` trying to take its own
lock.
That double lock is undefined behaviour and can cause a deadlock on the
mutex, although it appears rare that it actually does. If it does,
however, this eventually backs up into vote relaying during the idle
loop, which then stalls the idle loop so we stop sending out uptime
proofs (since that is also in the idle loop).
A simple fix here is to add the `true` argument, but on reconsideration
this extra argument to take or not take a lock is messy and error prone,
so this commit instead removes the second argument entirely and instead
documents which call must and must not hold a lock, getting rid of the
three methods (get_blink, has_blink, and add_existing_blink) that had
the `have_lock` argument. This ends up having only a small impact on
calling code - the vast majority of callers already hold a lock, and the
few that don't are easily adjusted.
2020-01-18 19:01:45 +01:00
bool tx_memory_pool : : has_blink ( const crypto : : hash & tx_hash ) const
2019-11-06 07:28:33 +01:00
{
return m_blinks . find ( tx_hash ) ! = m_blinks . end ( ) ;
}
2019-11-18 23:18:41 +01:00
2019-11-24 19:57:44 +01:00
void tx_memory_pool : : keep_missing_blinks ( std : : vector < crypto : : hash > & tx_hashes ) const
2019-11-18 23:18:41 +01:00
{
auto lock = blink_shared_lock ( ) ;
tx_hashes . erase (
std : : remove_if ( tx_hashes . begin ( ) , tx_hashes . end ( ) ,
[ this ] ( const crypto : : hash & tx_hash ) { return m_blinks . count ( tx_hash ) > 0 ; } ) ,
tx_hashes . end ( ) ) ;
}
std : : pair < std : : vector < crypto : : hash > , std : : vector < uint64_t > > tx_memory_pool : : get_blink_hashes_and_mined_heights ( ) const
{
std : : pair < std : : vector < crypto : : hash > , std : : vector < uint64_t > > hnh ;
2019-11-27 18:47:28 +01:00
auto & hashes = hnh . first ;
auto & heights = hnh . second ;
2019-11-18 23:18:41 +01:00
{
auto lock = blink_shared_lock ( ) ;
if ( ! m_blinks . empty ( ) )
{
2019-11-27 18:47:28 +01:00
hashes . reserve ( m_blinks . size ( ) ) ;
2019-11-18 23:18:41 +01:00
for ( auto & b : m_blinks )
2019-11-27 18:47:28 +01:00
hashes . push_back ( b . first ) ;
2019-11-18 23:18:41 +01:00
}
}
2019-11-25 05:22:02 +01:00
heights = m_blockchain . get_transactions_heights ( hashes ) ;
2019-11-27 18:47:28 +01:00
// Filter out (and delete from the blink pool) any blinks that are in immutable blocks
const uint64_t immutable_height = m_blockchain . get_immutable_height ( ) ;
size_t next_good = 0 ;
for ( size_t i = 0 ; i < hashes . size ( ) ; i + + )
{
2019-12-21 02:00:10 +01:00
if ( heights [ i ] > immutable_height | | heights [ i ] = = 0 /* unmined mempool blink */ )
2019-11-27 18:47:28 +01:00
{
// Swap elements into the "good" part of the list so that when we're we'll have divided the
// vector into [0, ..., next_good-1] elements containing the parts we want to return, and
// [next_good, ...] containing the elements to remove from blink storage.
if ( i ! = next_good )
{
using std : : swap ;
swap ( heights [ i ] , heights [ next_good ] ) ;
swap ( hashes [ i ] , hashes [ next_good ] ) ;
}
next_good + + ;
}
}
if ( next_good < hashes . size ( ) )
{
auto lock = blink_unique_lock ( ) ;
for ( size_t i = next_good ; i < hashes . size ( ) ; i + + )
m_blinks . erase ( hashes [ i ] ) ;
}
hashes . resize ( next_good ) ;
heights . resize ( next_good ) ;
2019-11-18 23:18:41 +01:00
return hnh ;
}
std : : map < uint64_t , crypto : : hash > tx_memory_pool : : get_blink_checksums ( ) const
{
std : : map < uint64_t , crypto : : hash > result ;
auto hnh = get_blink_hashes_and_mined_heights ( ) ;
auto & hashes = hnh . first ;
auto & heights = hnh . second ;
for ( size_t i = 0 ; i < hashes . size ( ) ; i + + )
{
auto it = result . lower_bound ( heights [ i ] ) ;
if ( it = = result . end ( ) | | it - > first ! = heights [ i ] )
result . emplace_hint ( it , heights [ i ] , hashes [ i ] ) ;
else
2020-07-16 00:28:01 +02:00
it - > second ^ = hashes [ i ] ;
2019-11-18 23:18:41 +01:00
}
return result ;
}
2019-11-06 07:28:33 +01:00
//---------------------------------------------------------------------------------
2019-11-18 23:18:41 +01:00
std : : vector < crypto : : hash > tx_memory_pool : : get_mined_blinks ( const std : : set < uint64_t > & want_heights ) const
2019-11-06 07:28:33 +01:00
{
2019-11-18 23:18:41 +01:00
std : : vector < crypto : : hash > result ;
Fix rare blink pool deadlock
cryptonote_protocol_handler calls `pool.get_blink(hash)` while already
holding a blink shared lock, which should have been
`pool.get_blink(hash, true)` to avoid `get_blink` trying to take its own
lock.
That double lock is undefined behaviour and can cause a deadlock on the
mutex, although it appears rare that it actually does. If it does,
however, this eventually backs up into vote relaying during the idle
loop, which then stalls the idle loop so we stop sending out uptime
proofs (since that is also in the idle loop).
A simple fix here is to add the `true` argument, but on reconsideration
this extra argument to take or not take a lock is messy and error prone,
so this commit instead removes the second argument entirely and instead
documents which call must and must not hold a lock, getting rid of the
three methods (get_blink, has_blink, and add_existing_blink) that had
the `have_lock` argument. This ends up having only a small impact on
calling code - the vast majority of callers already hold a lock, and the
few that don't are easily adjusted.
2020-01-18 19:01:45 +01:00
2019-11-18 23:18:41 +01:00
auto hnh = get_blink_hashes_and_mined_heights ( ) ;
auto & hashes = hnh . first ;
auto & heights = hnh . second ;
for ( size_t i = 0 ; i < heights . size ( ) ; i + + )
{
if ( want_heights . count ( heights [ i ] ) )
result . push_back ( hashes [ i ] ) ;
2019-11-06 07:28:33 +01:00
}
2019-11-18 23:18:41 +01:00
return result ;
2019-11-06 07:28:33 +01:00
}
2019-11-18 23:18:41 +01:00
2019-11-25 05:57:15 +01:00
//---------------------------------------------------------------------------------
bool tx_memory_pool : : remove_blink_conflicts ( const crypto : : hash & id , const std : : vector < crypto : : hash > & conflict_txs , uint64_t * blink_rollback_height )
{
2019-12-25 05:13:02 +01:00
auto bl_lock = blink_shared_lock ( std : : defer_lock ) ;
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
std : : unique_lock bc_lock { m_blockchain , std : : defer_lock } ;
2020-06-02 01:01:24 +02:00
std : : lock ( bl_lock , bc_lock ) ;
2019-12-25 05:13:02 +01:00
2019-11-25 05:57:15 +01:00
// Since this is a signed blink tx, we want to see if we can eject any existing mempool
// txes to make room.
// First check to see if any of the conflicting txes is itself an approved blink as a
// safety check (it shouldn't be possible if the network is functioning properly).
for ( const auto & tx_hash : conflict_txs )
{
if ( m_blinks . count ( tx_hash ) )
{
MERROR ( " Blink error: incoming blink tx " < < id < < " conflicts with another blink tx " < < tx_hash ) ;
return false ;
}
}
uint64_t rollback_height_needed = blink_rollback_height ? * blink_rollback_height : 0 ;
std : : vector < crypto : : hash > mempool_txs ;
// Next make sure none of the conflicting txes are mined in immutable blocks
auto immutable_height = m_blockchain . get_immutable_height ( ) ;
auto heights = m_blockchain . get_transactions_heights ( conflict_txs ) ;
for ( size_t i = 0 ; i < heights . size ( ) ; + + i )
{
2019-12-25 05:16:29 +01:00
MDEBUG ( " Conflicting tx " < < conflict_txs [ i ] < < ( heights [ i ] ? " mined at height " + std : : to_string ( heights [ i ] ) : " in mempool " ) ) ;
2019-11-25 05:57:15 +01:00
if ( ! heights [ i ] )
{
mempool_txs . push_back ( conflict_txs [ i ] ) ;
}
else if ( heights [ i ] > immutable_height & & blink_rollback_height )
{
2019-12-26 07:38:00 +01:00
if ( rollback_height_needed = = 0 | | rollback_height_needed > heights [ i ] )
rollback_height_needed = heights [ i ] ;
2019-11-25 05:57:15 +01:00
// else already set to something at least as early as this tx
}
else
return false ;
}
if ( ! mempool_txs . empty ( ) )
{
2019-12-25 05:13:02 +01:00
LockedTXN txnlock ( m_blockchain ) ;
2019-11-25 05:57:15 +01:00
for ( auto & tx : mempool_txs )
{
MWARNING ( " Removing conflicting tx " < < tx < < " from mempool for incoming blink tx " < < id ) ;
if ( ! remove_tx ( tx ) )
{
MERROR ( " Internal error: Unable to clear conflicting tx " < < tx < < " from mempool for incoming blink tx " < < id ) ;
return false ;
}
}
2019-12-25 05:13:02 +01:00
txnlock . commit ( ) ;
2019-11-25 05:57:15 +01:00
}
if ( blink_rollback_height & & rollback_height_needed < * blink_rollback_height )
2019-12-26 07:38:00 +01:00
{
MINFO ( " Incoming blink tx requires a rollback to the " < < rollback_height_needed < < " to un-mine conflicting transactions " ) ;
2019-11-25 05:57:15 +01:00
* blink_rollback_height = rollback_height_needed ;
2019-12-26 07:38:00 +01:00
}
2019-11-25 05:57:15 +01:00
return true ;
}
2019-11-06 07:28:33 +01:00
//---------------------------------------------------------------------------------
2018-07-18 23:24:53 +02:00
size_t tx_memory_pool : : get_txpool_weight ( ) const
2018-01-29 22:20:24 +01:00
{
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
std : : unique_lock lock { m_transactions_lock } ;
2018-07-18 23:24:53 +02:00
return m_txpool_weight ;
2018-01-29 22:20:24 +01:00
}
//---------------------------------------------------------------------------------
2018-07-18 23:24:53 +02:00
void tx_memory_pool : : set_txpool_max_weight ( size_t bytes )
2018-01-29 22:20:24 +01:00
{
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
std : : unique_lock lock { m_transactions_lock } ;
2018-07-18 23:24:53 +02:00
m_txpool_max_weight = bytes ;
2018-01-29 22:20:24 +01:00
}
//---------------------------------------------------------------------------------
2019-11-24 19:45:09 +01:00
bool tx_memory_pool : : remove_tx ( const crypto : : hash & txid , const txpool_tx_meta_t * meta , const sorted_tx_container : : iterator * stc_it )
2018-01-29 22:20:24 +01:00
{
2019-11-24 19:45:09 +01:00
const auto it = stc_it ? * stc_it : find_tx_in_sorted_container ( txid ) ;
if ( it = = m_txs_by_fee_and_receive_time . end ( ) )
{
MERROR ( " Failed to find tx in txpool sorted list " ) ;
return false ;
}
2018-01-29 22:20:24 +01:00
2019-11-24 19:45:09 +01:00
cryptonote : : blobdata tx_blob = m_blockchain . get_txpool_tx_blob ( txid ) ;
cryptonote : : transaction_prefix tx ;
if ( ! parse_and_validate_tx_prefix_from_blob ( tx_blob , tx ) )
2018-10-09 03:31:08 +02:00
{
2019-11-24 19:45:09 +01:00
MERROR ( " Failed to parse tx from txpool " ) ;
return false ;
}
txpool_tx_meta_t lookup_meta ;
if ( ! meta )
{
if ( m_blockchain . get_txpool_tx_meta ( txid , lookup_meta ) )
meta = & lookup_meta ;
else
2018-10-22 06:16:28 +02:00
{
2019-11-24 19:45:09 +01:00
MERROR ( " Failed to find tx in txpool " ) ;
2018-10-22 06:16:28 +02:00
return false ;
}
2019-11-24 19:45:09 +01:00
}
2018-10-22 06:16:28 +02:00
2019-11-24 19:45:09 +01:00
// remove first, in case this throws, so key images aren't removed
const uint64_t tx_fee = std : : get < 1 > ( it - > first ) ;
MINFO ( " Removing tx " < < txid < < " from txpool: weight: " < < meta - > weight < < " , fee/byte: " < < tx_fee ) ;
m_blockchain . remove_txpool_tx ( txid ) ;
m_txpool_weight - = meta - > weight ;
remove_transaction_keyimages ( tx , txid ) ;
m_txs_by_fee_and_receive_time . erase ( it ) ;
2018-10-09 03:31:08 +02:00
2019-11-24 19:45:09 +01:00
return true ;
}
//---------------------------------------------------------------------------------
2019-11-25 05:57:15 +01:00
void tx_memory_pool : : prune ( const crypto : : hash & skip )
2019-11-24 19:45:09 +01:00
{
2019-11-24 20:02:35 +01:00
auto blink_lock = blink_shared_lock ( std : : defer_lock ) ;
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
std : : unique_lock tx_lock { * this , std : : defer_lock } ;
std : : unique_lock bc_lock { m_blockchain , std : : defer_lock } ;
2020-06-02 01:01:24 +02:00
std : : lock ( blink_lock , tx_lock , bc_lock ) ;
2019-11-24 19:45:09 +01:00
LockedTXN lock ( m_blockchain ) ;
bool changed = false ;
2018-10-09 03:31:08 +02:00
2019-11-24 19:45:09 +01:00
// Tries checking conditions for pruning and, if appropriate, removing the tx.
// Returns false on failure, true for no prune wanted or a successful prune.
2019-11-25 05:57:15 +01:00
auto try_pruning = [ this , & skip , & changed ] ( auto & it , bool forward ) - > bool {
2018-08-21 09:01:19 +02:00
try
{
const crypto : : hash & txid = it - > second ;
txpool_tx_meta_t meta ;
if ( ! m_blockchain . get_txpool_tx_meta ( txid , meta ) )
{
MERROR ( " Failed to find tx in txpool " ) ;
2019-11-24 19:45:09 +01:00
return false ;
2018-08-21 09:01:19 +02:00
}
2019-11-24 19:45:09 +01:00
auto del_it = forward ? it + + : it - - ;
2018-08-21 09:01:19 +02:00
// don't prune the kept_by_block ones, they're likely added because we're adding a block with those
2019-11-24 20:02:35 +01:00
// don't prune blink txes
2019-11-25 05:57:15 +01:00
// don't prune the one we just added
Fix rare blink pool deadlock
cryptonote_protocol_handler calls `pool.get_blink(hash)` while already
holding a blink shared lock, which should have been
`pool.get_blink(hash, true)` to avoid `get_blink` trying to take its own
lock.
That double lock is undefined behaviour and can cause a deadlock on the
mutex, although it appears rare that it actually does. If it does,
however, this eventually backs up into vote relaying during the idle
loop, which then stalls the idle loop so we stop sending out uptime
proofs (since that is also in the idle loop).
A simple fix here is to add the `true` argument, but on reconsideration
this extra argument to take or not take a lock is messy and error prone,
so this commit instead removes the second argument entirely and instead
documents which call must and must not hold a lock, getting rid of the
three methods (get_blink, has_blink, and add_existing_blink) that had
the `have_lock` argument. This ends up having only a small impact on
calling code - the vast majority of callers already hold a lock, and the
few that don't are easily adjusted.
2020-01-18 19:01:45 +01:00
if ( meta . kept_by_block | | this - > has_blink ( txid ) | | txid = = skip )
2019-11-24 19:45:09 +01:00
return true ;
2018-10-09 03:31:08 +02:00
2019-11-29 05:49:33 +01:00
if ( this - > remove_tx ( txid , & meta , & del_it ) )
2018-08-21 09:01:19 +02:00
{
2019-11-24 19:45:09 +01:00
changed = true ;
return true ;
2018-08-21 09:01:19 +02:00
}
2019-11-24 19:45:09 +01:00
return false ;
2018-08-21 09:01:19 +02:00
}
catch ( const std : : exception & e )
{
MERROR ( " Error while pruning txpool: " < < e . what ( ) ) ;
2019-11-24 19:45:09 +01:00
return false ;
2018-08-21 09:01:19 +02:00
}
2019-11-24 19:45:09 +01:00
} ;
const auto unexpired = std : : time ( nullptr ) - MEMPOOL_PRUNE_NON_STANDARD_TX_LIFETIME ;
for ( auto it = m_txs_by_fee_and_receive_time . begin ( ) ; it ! = m_txs_by_fee_and_receive_time . end ( ) ; )
{
const bool is_standard_tx = ! std : : get < 0 > ( it - > first ) ;
const time_t receive_time = std : : get < 2 > ( it - > first ) ;
if ( is_standard_tx | | receive_time > = unexpired )
break ;
if ( ! try_pruning ( it , true /*forward*/ ) )
return ;
2018-08-21 09:01:19 +02:00
}
2018-01-29 22:20:24 +01:00
// this will never remove the first one, but we don't care
Relax deregistration rules
The replaces the deregistration mechanism with a new state change
mechanism (beginning at the v12 fork) which can change a service node's
network status via three potential values (and is extensible in the
future to handle more):
- deregistered -- this is the same as the existing deregistration; the
SN is instantly removed from the SN list.
- decommissioned -- this is a sort of temporary deregistration: your SN
remains in the service node list, but is removed from the rewards list
and from any network duties.
- recommissioned -- this tx is sent by a quorum if they observe a
decommissioned SN sending uptime proofs again. Upon reception, the SN
is reactivated and put on the end of the reward list.
Since this is broadening the quorum use, this also renames the relevant
quorum to a "obligations" quorum (since it validates SN obligations),
while the transactions are "state_change" transactions (since they
change the state of a registered SN).
The new parameters added to service_node_rules.h control how this works:
// Service node decommissioning: as service nodes stay up they earn "credits" (measured in blocks)
// towards a future outage. A new service node starts out with INITIAL_CREDIT, and then builds up
// CREDIT_PER_DAY for each day the service node remains active up to a maximum of
// DECOMMISSION_MAX_CREDIT.
//
// If a service node stops sending uptime proofs, a quorum will consider whether the service node
// has built up enough credits (at least MINIMUM): if so, instead of submitting a deregistration,
// it instead submits a decommission. This removes the service node from the list of active
// service nodes both for rewards and for any active network duties. If the service node comes
// back online (i.e. starts sending the required performance proofs again) before the credits run
// out then a quorum will reinstate the service node using a recommission transaction, which adds
// the service node back to the bottom of the service node reward list, and resets its accumulated
// credits to 0. If it does not come back online within the required number of blocks (i.e. the
// accumulated credit at the point of decommissioning) then a quorum will send a permanent
// deregistration transaction to the network, starting a 30-day deregistration count down.
This commit currently includes values (which are not necessarily
finalized):
- 8 hours (240 blocks) of credit required for activation of a
decommission (rather than a deregister)
- 0 initial credits at registration
- a maximum of 24 hours (720 blocks) of credits
- credits accumulate at a rate that you hit 24 hours of credits after 30
days of operation.
Miscellaneous other details of this PR:
- a new TX extra tag is used for the state change (including
deregistrations). The old extra tag has no version or type tag, so
couldn't be reused. The data in the new tag is slightly more
efficiently packed than the old deregistration transaction, so it gets
used for deregistrations (starting at the v12 fork) as well.
- Correct validator/worker selection required generalizing the shuffle
function to be able to shuffle just part of a vector. This lets us
stick any down service nodes at the end of the potential list, then
select validators by only shuffling the part of the index vector that
contains active service indices. Once the validators are selected, the
remainder of the list (this time including decommissioned SN indices) is
shuffled to select quorum workers to check, thus allowing decommisioned
nodes to be randomly included in the nodes to check without being
selected as a validator.
- Swarm recalculation was not quite right: swarms were recalculated on
SN registrations, even if those registrations were include shared node
registrations, but *not* recalculated on stakes. Starting with the
upgrade this behaviour is fixed (swarms aren't actually used currently
and aren't consensus-relevant so recalculating early won't hurt
anything).
- Details on decomm/dereg are added to RPC info and print_sn/print_sn_status
- Slightly improves the % of reward output in the print_sn output by
rounding it to two digits, and reserves space in the output string to
avoid excessive reallocations.
- Adds various debugging at higher debug levels to quorum voting (into
all of voting itself, vote transmission, and vote reception).
- Reset service node list internal data structure version to 0. The SN
list has to be rescanned anyway at upgrade (its size has changed), so we
might as well reset the version and remove the version-dependent
serialization code. (Note that the affected code here is for SN states
in lmdb storage, not for SN-to-SN communication serialization).
2019-06-18 23:57:02 +02:00
auto it = m_txs_by_fee_and_receive_time . end ( ) ;
if ( it ! = m_txs_by_fee_and_receive_time . begin ( ) )
it = std : : prev ( it ) ;
2019-11-26 08:23:51 +01:00
while ( m_txpool_weight > m_txpool_max_weight & & it ! = m_txs_by_fee_and_receive_time . begin ( ) )
2018-01-29 22:20:24 +01:00
{
2019-11-24 19:45:09 +01:00
if ( ! try_pruning ( it , false /*forward*/ ) )
2018-01-29 22:20:24 +01:00
return ;
}
2019-06-26 06:00:05 +02:00
lock . commit ( ) ;
2016-04-17 12:04:01 +02:00
if ( changed )
+ + m_cookie ;
2019-11-26 08:23:51 +01:00
if ( m_txpool_weight > m_txpool_max_weight )
MINFO ( " Pool weight after pruning is still larger than limit: " < < m_txpool_weight < < " / " < < m_txpool_max_weight ) ;
2018-01-29 22:20:24 +01:00
}
//---------------------------------------------------------------------------------
2018-09-16 20:30:39 +02:00
bool tx_memory_pool : : insert_key_images ( const transaction_prefix & tx , const crypto : : hash & id , bool kept_by_block )
2017-05-14 15:06:55 +02:00
{
for ( const auto & in : tx . vin )
{
2020-06-02 05:47:20 +02:00
CHECKED_GET_SPECIFIC_VARIANT ( in , txin_to_key , txin , false ) ;
2017-05-14 15:06:55 +02:00
std : : unordered_set < crypto : : hash > & kei_image_set = m_spent_key_images [ txin . k_image ] ;
CHECK_AND_ASSERT_MES ( kept_by_block | | kei_image_set . size ( ) = = 0 , false , " internal error: kept_by_block= " < < kept_by_block
2020-04-03 18:30:11 +02:00
< < " , kei_image_set.size()= " < < kei_image_set . size ( ) < < " \n txin.k_image= " < < txin . k_image
< < " \n tx_id= " < < id ) ;
2017-05-14 15:06:55 +02:00
auto ins_res = kei_image_set . insert ( id ) ;
CHECK_AND_ASSERT_MES ( ins_res . second , false , " internal error: try to insert duplicate iterator in key_image set " ) ;
}
2016-04-17 12:04:01 +02:00
+ + m_cookie ;
2017-05-14 15:06:55 +02:00
return true ;
}
//---------------------------------------------------------------------------------
2016-03-25 01:03:02 +01:00
//FIXME: Can return early before removal of all of the key images.
// At the least, need to make sure that a false return here
// is treated properly. Should probably not return early, however.
2018-09-16 20:30:39 +02:00
bool tx_memory_pool : : remove_transaction_keyimages ( const transaction_prefix & tx , const crypto : : hash & actual_hash )
2014-03-03 23:07:58 +01:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2015-12-14 05:54:39 +01:00
// ND: Speedup
2017-01-22 21:38:10 +01:00
for ( const txin_v & vi : tx . vin )
2014-03-03 23:07:58 +01:00
{
2020-06-02 05:47:20 +02:00
CHECKED_GET_SPECIFIC_VARIANT ( vi , txin_to_key , txin , false ) ;
2014-03-03 23:07:58 +01:00
auto it = m_spent_key_images . find ( txin . k_image ) ;
2020-04-03 18:30:11 +02:00
CHECK_AND_ASSERT_MES ( it ! = m_spent_key_images . end ( ) , false , " failed to find transaction input in key images. img= " < < txin . k_image
< < " \n transaction id = " < < actual_hash ) ;
2014-03-03 23:07:58 +01:00
std : : unordered_set < crypto : : hash > & key_image_set = it - > second ;
2020-04-03 18:30:11 +02:00
CHECK_AND_ASSERT_MES ( key_image_set . size ( ) , false , " empty key_image set, img= " < < txin . k_image
< < " \n transaction id = " < < actual_hash ) ;
2014-03-03 23:07:58 +01:00
** CHANGES ARE EXPERIMENTAL (FOR TESTING ONLY)
Bockchain:
1. Optim: Multi-thread long-hash computation when encountering groups of blocks.
2. Optim: Cache verified txs and return result from cache instead of re-checking whenever possible.
3. Optim: Preload output-keys when encoutering groups of blocks. Sort by amount and global-index before bulk querying database and multi-thread when possible.
4. Optim: Disable double spend check on block verification, double spend is already detected when trying to add blocks.
5. Optim: Multi-thread signature computation whenever possible.
6. Patch: Disable locking (recursive mutex) on called functions from check_tx_inputs which causes slowdowns (only seems to happen on ubuntu/VMs??? Reason: TBD)
7. Optim: Removed looped full-tx hash computation when retrieving transactions from pool (???).
8. Optim: Cache difficulty/timestamps (735 blocks) for next-difficulty calculations so that only 2 db reads per new block is needed when a new block arrives (instead of 1470 reads).
Berkeley-DB:
1. Fix: 32-bit data errors causing wrong output global indices and failure to send blocks to peers (etc).
2. Fix: Unable to pop blocks on reorganize due to transaction errors.
3. Patch: Large number of transaction aborts when running multi-threaded bulk queries.
4. Patch: Insufficient locks error when running full sync.
5. Patch: Incorrect db stats when returning from an immediate exit from "pop block" operation.
6. Optim: Add bulk queries to get output global indices.
7. Optim: Modified output_keys table to store public_key+unlock_time+height for single transaction lookup (vs 3)
8. Optim: Used output_keys table retrieve public_keys instead of going through output_amounts->output_txs+output_indices->txs->output:public_key
9. Optim: Added thread-safe buffers used when multi-threading bulk queries.
10. Optim: Added support for nosync/write_nosync options for improved performance (*see --db-sync-mode option for details)
11. Mod: Added checkpoint thread and auto-remove-logs option.
12. *Now usable on 32-bit systems like RPI2.
LMDB:
1. Optim: Added custom comparison for 256-bit key tables (minor speed-up, TBD: get actual effect)
2. Optim: Modified output_keys table to store public_key+unlock_time+height for single transaction lookup (vs 3)
3. Optim: Used output_keys table retrieve public_keys instead of going through output_amounts->output_txs+output_indices->txs->output:public_key
4. Optim: Added support for sync/writemap options for improved performance (*see --db-sync-mode option for details)
5. Mod: Auto resize to +1GB instead of multiplier x1.5
ETC:
1. Minor optimizations for slow-hash for ARM (RPI2). Incomplete.
2. Fix: 32-bit saturation bug when computing next difficulty on large blocks.
[PENDING ISSUES]
1. Berkely db has a very slow "pop-block" operation. This is very noticeable on the RPI2 as it sometimes takes > 10 MINUTES to pop a block during reorganization.
This does not happen very often however, most reorgs seem to take a few seconds but it possibly depends on the number of outputs present. TBD.
2. Berkeley db, possible bug "unable to allocate memory". TBD.
[NEW OPTIONS] (*Currently all enabled for testing purposes)
1. --fast-block-sync arg=[0:1] (default: 1)
a. 0 = Compute long hash per block (may take a while depending on CPU)
b. 1 = Skip long-hash and verify blocks based on embedded known good block hashes (faster, minimal CPU dependence)
2. --db-sync-mode arg=[[safe|fast|fastest]:[sync|async]:[nblocks_per_sync]] (default: fastest:async:1000)
a. safe = fdatasync/fsync (or equivalent) per stored block. Very slow, but safest option to protect against power-out/crash conditions.
b. fast/fastest = Enables asynchronous fdatasync/fsync (or equivalent). Useful for battery operated devices or STABLE systems with UPS and/or systems with battery backed write cache/solid state cache.
Fast - Write meta-data but defer data flush.
Fastest - Defer meta-data and data flush.
Sync - Flush data after nblocks_per_sync and wait.
Async - Flush data after nblocks_per_sync but do not wait for the operation to finish.
3. --prep-blocks-threads arg=[n] (default: 4 or system max threads, whichever is lower)
Max number of threads to use when computing long-hash in groups.
4. --show-time-stats arg=[0:1] (default: 1)
Show benchmark related time stats.
5. --db-auto-remove-logs arg=[0:1] (default: 1)
For berkeley-db only. Auto remove logs if enabled.
**Note: lmdb and berkeley-db have changes to the tables and are not compatible with official git head version.
At the moment, you need a full resync to use this optimized version.
[PERFORMANCE COMPARISON]
**Some figures are approximations only.
Using a baseline machine of an i7-2600K+SSD+(with full pow computation):
1. The optimized lmdb/blockhain core can process blocks up to 585K for ~1.25 hours + download time, so it usually takes 2.5 hours to sync the full chain.
2. The current head with memory can process blocks up to 585K for ~4.2 hours + download time, so it usually takes 5.5 hours to sync the full chain.
3. The current head with lmdb can process blocks up to 585K for ~32 hours + download time and usually takes 36 hours to sync the full chain.
Averate procesing times (with full pow computation):
lmdb-optimized:
1. tx_ave = 2.5 ms / tx
2. block_ave = 5.87 ms / block
memory-official-repo:
1. tx_ave = 8.85 ms / tx
2. block_ave = 19.68 ms / block
lmdb-official-repo (0f4a036437fd41a5498ee5e74e2422ea6177aa3e)
1. tx_ave = 47.8 ms / tx
2. block_ave = 64.2 ms / block
**Note: The following data denotes processing times only (does not include p2p download time)
lmdb-optimized processing times (with full pow computation):
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 1.25 hours processing time (--db-sync-mode=fastest:async:1000).
2. Laptop, Dual-core / 4-threads U4200 (3Mb) - 4.90 hours processing time (--db-sync-mode=fastest:async:1000).
3. Embedded, Quad-core / 4-threads Z3735F (2x1Mb) - 12.0 hours processing time (--db-sync-mode=fastest:async:1000).
lmdb-optimized processing times (with per-block-checkpoint)
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 10 minutes processing time (--db-sync-mode=fastest:async:1000).
berkeley-db optimized processing times (with full pow computation)
1. Desktop, Quad-core / 8-threads 2600k (8Mb) - 1.8 hours processing time (--db-sync-mode=fastest:async:1000).
2. RPI2. Improved from estimated 3 months(???) into 2.5 days (*Need 2AMP supply + Clock:1Ghz + [usb+ssd] to achieve this speed) (--db-sync-mode=fastest:async:1000).
berkeley-db optimized processing times (with per-block-checkpoint)
1. RPI2. 12-15 hours (*Need 2AMP supply + Clock:1Ghz + [usb+ssd] to achieve this speed) (--db-sync-mode=fastest:async:1000).
2015-07-10 22:09:32 +02:00
auto it_in_set = key_image_set . find ( actual_hash ) ;
2020-04-03 18:30:11 +02:00
CHECK_AND_ASSERT_MES ( it_in_set ! = key_image_set . end ( ) , false , " transaction id not found in key_image set, img= " < < txin . k_image
< < " \n transaction id = " < < actual_hash ) ;
2014-03-03 23:07:58 +01:00
key_image_set . erase ( it_in_set ) ;
if ( ! key_image_set . size ( ) )
{
//it is now empty hash container for this key_image
m_spent_key_images . erase ( it ) ;
}
}
2016-04-17 12:04:01 +02:00
+ + m_cookie ;
2014-03-03 23:07:58 +01:00
return true ;
}
//---------------------------------------------------------------------------------
2018-11-11 15:51:03 +01:00
bool tx_memory_pool : : take_tx ( const crypto : : hash & id , transaction & tx , cryptonote : : blobdata & txblob , size_t & tx_weight , uint64_t & fee , bool & relayed , bool & do_not_relay , bool & double_spend_seen )
2014-03-03 23:07:58 +01:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2014-03-03 23:07:58 +01:00
2015-05-14 02:27:06 +02:00
auto sorted_it = find_tx_in_sorted_container ( id ) ;
2015-04-30 07:02:12 +02:00
2017-05-14 15:06:55 +02:00
try
{
LockedTXN lock ( m_blockchain ) ;
2017-12-14 18:09:30 +01:00
txpool_tx_meta_t meta ;
if ( ! m_blockchain . get_txpool_tx_meta ( id , meta ) )
{
MERROR ( " Failed to find tx in txpool " ) ;
return false ;
}
2018-11-11 15:51:03 +01:00
txblob = m_blockchain . get_txpool_tx_blob ( id ) ;
2018-10-31 14:52:16 +01:00
auto ci = m_parsed_tx_cache . find ( id ) ;
if ( ci ! = m_parsed_tx_cache . end ( ) )
{
tx = ci - > second ;
}
else if ( ! parse_and_validate_tx_from_blob ( txblob , tx ) )
2017-05-14 15:06:55 +02:00
{
MERROR ( " Failed to parse tx from txpool " ) ;
return false ;
}
2018-12-05 21:34:10 +01:00
else
{
tx . set_hash ( id ) ;
}
2018-07-18 23:24:53 +02:00
tx_weight = meta . weight ;
2017-05-14 15:06:55 +02:00
fee = meta . fee ;
relayed = meta . relayed ;
do_not_relay = meta . do_not_relay ;
2017-09-22 14:57:20 +02:00
double_spend_seen = meta . double_spend_seen ;
2017-05-14 15:06:55 +02:00
// remove first, in case this throws, so key images aren't removed
m_blockchain . remove_txpool_tx ( id ) ;
2018-07-18 23:24:53 +02:00
m_txpool_weight - = tx_weight ;
2018-09-16 20:30:39 +02:00
remove_transaction_keyimages ( tx , id ) ;
2019-06-26 06:00:05 +02:00
lock . commit ( ) ;
2017-05-14 15:06:55 +02:00
}
catch ( const std : : exception & e )
{
MERROR ( " Failed to remove tx from txpool: " < < e . what ( ) ) ;
return false ;
}
2018-10-31 15:43:13 +01:00
if ( sorted_it ! = m_txs_by_fee_and_receive_time . end ( ) )
m_txs_by_fee_and_receive_time . erase ( sorted_it ) ;
2016-04-17 12:04:01 +02:00
+ + m_cookie ;
2014-03-03 23:07:58 +01:00
return true ;
}
//---------------------------------------------------------------------------------
2014-06-15 09:48:13 +02:00
void tx_memory_pool : : on_idle ( )
{
m_remove_stuck_tx_interval . do_call ( [ this ] ( ) { return remove_stuck_transactions ( ) ; } ) ;
}
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
void tx_memory_pool : : add_notify ( std : : function < void ( const crypto : : hash & , const transaction & , const std : : string & , const tx_pool_options & ) > notify )
{
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
std : : unique_lock lock { m_transactions_lock } ;
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
m_tx_notify . push_back ( std : : move ( notify ) ) ;
}
2014-06-15 09:48:13 +02:00
//---------------------------------------------------------------------------------
2015-05-14 02:27:06 +02:00
sorted_tx_container : : iterator tx_memory_pool : : find_tx_in_sorted_container ( const crypto : : hash & id ) const
{
2017-01-13 16:08:37 +01:00
return std : : find_if ( m_txs_by_fee_and_receive_time . begin ( ) , m_txs_by_fee_and_receive_time . end ( )
2015-05-14 02:27:06 +02:00
, [ & ] ( const sorted_tx_container : : value_type & a ) {
return a . second = = id ;
}
) ;
}
//---------------------------------------------------------------------------------
2016-03-25 01:03:02 +01:00
//TODO: investigate whether boolean return is appropriate
2014-06-15 09:48:13 +02:00
bool tx_memory_pool : : remove_stuck_transactions ( )
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2018-09-16 20:30:39 +02:00
std : : list < std : : pair < crypto : : hash , uint64_t > > remove ;
2017-05-14 15:06:55 +02:00
m_blockchain . for_all_txpool_txes ( [ this , & remove ] ( const crypto : : hash & txid , const txpool_tx_meta_t & meta , const cryptonote : : blobdata * ) {
uint64_t tx_age = time ( nullptr ) - meta . receive_time ;
2014-06-15 09:48:13 +02:00
2017-05-14 15:06:55 +02:00
if ( ( tx_age > CRYPTONOTE_MEMPOOL_TX_LIVETIME & & ! meta . kept_by_block ) | |
( tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME & & meta . kept_by_block ) )
2014-06-15 09:48:13 +02:00
{
2017-05-14 15:06:55 +02:00
LOG_PRINT_L1 ( " Tx " < < txid < < " removed from tx pool due to outdated, age: " < < tx_age ) ;
auto sorted_it = find_tx_in_sorted_container ( txid ) ;
2017-01-13 16:08:37 +01:00
if ( sorted_it = = m_txs_by_fee_and_receive_time . end ( ) )
2015-05-14 02:27:06 +02:00
{
2017-05-14 15:06:55 +02:00
LOG_PRINT_L1 ( " Removing tx " < < txid < < " from tx pool, but it was not found in the sorted txs container! " ) ;
2015-05-14 02:27:06 +02:00
}
else
{
2017-01-13 16:08:37 +01:00
m_txs_by_fee_and_receive_time . erase ( sorted_it ) ;
2015-05-14 02:27:06 +02:00
}
2017-05-14 15:06:55 +02:00
m_timed_out_transactions . insert ( txid ) ;
2018-09-16 20:30:39 +02:00
remove . push_back ( std : : make_pair ( txid , meta . weight ) ) ;
2017-05-14 15:06:55 +02:00
}
return true ;
2017-11-08 13:06:41 +01:00
} , false ) ;
2017-05-14 15:06:55 +02:00
if ( ! remove . empty ( ) )
{
LockedTXN lock ( m_blockchain ) ;
2018-09-16 20:30:39 +02:00
for ( const std : : pair < crypto : : hash , uint64_t > & entry : remove )
2017-05-14 15:06:55 +02:00
{
2018-09-16 20:30:39 +02:00
const crypto : : hash & txid = entry . first ;
2017-05-14 15:06:55 +02:00
try
{
cryptonote : : blobdata bd = m_blockchain . get_txpool_tx_blob ( txid ) ;
2018-09-16 20:30:39 +02:00
cryptonote : : transaction_prefix tx ;
if ( ! parse_and_validate_tx_prefix_from_blob ( bd , tx ) )
2017-05-14 15:06:55 +02:00
{
MERROR ( " Failed to parse tx from txpool " ) ;
// continue
}
else
{
// remove first, so we only remove key images if the tx removal succeeds
m_blockchain . remove_txpool_tx ( txid ) ;
2018-09-16 20:30:39 +02:00
m_txpool_weight - = entry . second ;
remove_transaction_keyimages ( tx , txid ) ;
2017-05-14 15:06:55 +02:00
}
}
catch ( const std : : exception & e )
{
MWARNING ( " Failed to remove stuck transaction: " < < txid ) ;
// ignore error
}
}
2019-06-26 06:00:05 +02:00
lock . commit ( ) ;
2016-04-17 12:04:01 +02:00
+ + m_cookie ;
2014-06-15 09:48:13 +02:00
}
return true ;
}
//---------------------------------------------------------------------------------
2016-03-25 01:03:02 +01:00
//TODO: investigate whether boolean return is appropriate
2018-04-16 01:16:02 +02:00
bool tx_memory_pool : : get_relayable_transactions ( std : : vector < std : : pair < crypto : : hash , cryptonote : : blobdata > > & txs ) const
2015-11-21 01:26:48 +01:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2017-05-14 15:06:55 +02:00
const uint64_t now = time ( NULL ) ;
2018-04-16 01:16:02 +02:00
txs . reserve ( m_blockchain . get_txpool_tx_count ( ) ) ;
2017-05-14 15:06:55 +02:00
m_blockchain . for_all_txpool_txes ( [ this , now , & txs ] ( const crypto : : hash & txid , const txpool_tx_meta_t & meta , const cryptonote : : blobdata * ) {
2019-11-06 07:28:33 +01:00
if ( ! meta . do_not_relay & & ( ! meta . relayed | | now - meta . last_relayed_time > get_relay_delay ( now , meta . receive_time ) ) )
2015-11-21 01:26:48 +01:00
{
// if the tx is older than half the max lifetime, we don't re-relay it, to avoid a problem
// mentioned by smooth where nodes would flush txes at slightly different times, causing
// flushed txes to be re-added when received from a node which was just about to flush it
2017-05-14 15:06:55 +02:00
uint64_t max_age = meta . kept_by_block ? CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME : CRYPTONOTE_MEMPOOL_TX_LIVETIME ;
if ( now - meta . receive_time < = max_age / 2 )
2015-11-21 01:26:48 +01:00
{
2017-05-14 15:06:55 +02:00
try
{
cryptonote : : blobdata bd = m_blockchain . get_txpool_tx_blob ( txid ) ;
Service Node Deregister Part 5 (#89)
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* core, service_node_list: separated address from service node pubkey
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* Store service node lists for the duration of deregister lifetimes
* Quorum min/max bug, sort node list, fix node to test list
* Change quorum to store acc pub address, fix oob bug
* Code review for expiring votes, acc keys to pub_key, improve err msgs
* Add early out for is_deregistration_tx and protect against quorum changes
* Remove debug code, fix segfault
* Remove irrelevant check for tx v3 in blockchain, fix >= height for pruning quorum states
Incorrect assumption that a transaction can be kept in the chain if it could
eventually become invalid, because if it were the chain would be split and
eventually these transaction would be dropped. But also that we should not
override the pre-existing logic which handles this case anyway.
2018-07-18 04:42:47 +02:00
if ( meta . fee = = 0 )
{
cryptonote : : transaction tx ;
2018-09-04 11:40:04 +02:00
if ( ! cryptonote : : parse_and_validate_tx_from_blob ( bd , tx ) )
Service Node Deregister Part 5 (#89)
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* core, service_node_list: separated address from service node pubkey
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* Store service node lists for the duration of deregister lifetimes
* Quorum min/max bug, sort node list, fix node to test list
* Change quorum to store acc pub address, fix oob bug
* Code review for expiring votes, acc keys to pub_key, improve err msgs
* Add early out for is_deregistration_tx and protect against quorum changes
* Remove debug code, fix segfault
* Remove irrelevant check for tx v3 in blockchain, fix >= height for pruning quorum states
Incorrect assumption that a transaction can be kept in the chain if it could
eventually become invalid, because if it were the chain would be split and
eventually these transaction would be dropped. But also that we should not
override the pre-existing logic which handles this case anyway.
2018-07-18 04:42:47 +02:00
{
2018-09-04 11:40:04 +02:00
LOG_PRINT_L1 ( " TX in pool could not be parsed from blob, txid: " < < txid ) ;
return true ;
}
Relax deregistration rules
The replaces the deregistration mechanism with a new state change
mechanism (beginning at the v12 fork) which can change a service node's
network status via three potential values (and is extensible in the
future to handle more):
- deregistered -- this is the same as the existing deregistration; the
SN is instantly removed from the SN list.
- decommissioned -- this is a sort of temporary deregistration: your SN
remains in the service node list, but is removed from the rewards list
and from any network duties.
- recommissioned -- this tx is sent by a quorum if they observe a
decommissioned SN sending uptime proofs again. Upon reception, the SN
is reactivated and put on the end of the reward list.
Since this is broadening the quorum use, this also renames the relevant
quorum to a "obligations" quorum (since it validates SN obligations),
while the transactions are "state_change" transactions (since they
change the state of a registered SN).
The new parameters added to service_node_rules.h control how this works:
// Service node decommissioning: as service nodes stay up they earn "credits" (measured in blocks)
// towards a future outage. A new service node starts out with INITIAL_CREDIT, and then builds up
// CREDIT_PER_DAY for each day the service node remains active up to a maximum of
// DECOMMISSION_MAX_CREDIT.
//
// If a service node stops sending uptime proofs, a quorum will consider whether the service node
// has built up enough credits (at least MINIMUM): if so, instead of submitting a deregistration,
// it instead submits a decommission. This removes the service node from the list of active
// service nodes both for rewards and for any active network duties. If the service node comes
// back online (i.e. starts sending the required performance proofs again) before the credits run
// out then a quorum will reinstate the service node using a recommission transaction, which adds
// the service node back to the bottom of the service node reward list, and resets its accumulated
// credits to 0. If it does not come back online within the required number of blocks (i.e. the
// accumulated credit at the point of decommissioning) then a quorum will send a permanent
// deregistration transaction to the network, starting a 30-day deregistration count down.
This commit currently includes values (which are not necessarily
finalized):
- 8 hours (240 blocks) of credit required for activation of a
decommission (rather than a deregister)
- 0 initial credits at registration
- a maximum of 24 hours (720 blocks) of credits
- credits accumulate at a rate that you hit 24 hours of credits after 30
days of operation.
Miscellaneous other details of this PR:
- a new TX extra tag is used for the state change (including
deregistrations). The old extra tag has no version or type tag, so
couldn't be reused. The data in the new tag is slightly more
efficiently packed than the old deregistration transaction, so it gets
used for deregistrations (starting at the v12 fork) as well.
- Correct validator/worker selection required generalizing the shuffle
function to be able to shuffle just part of a vector. This lets us
stick any down service nodes at the end of the potential list, then
select validators by only shuffling the part of the index vector that
contains active service indices. Once the validators are selected, the
remainder of the list (this time including decommissioned SN indices) is
shuffled to select quorum workers to check, thus allowing decommisioned
nodes to be randomly included in the nodes to check without being
selected as a validator.
- Swarm recalculation was not quite right: swarms were recalculated on
SN registrations, even if those registrations were include shared node
registrations, but *not* recalculated on stakes. Starting with the
upgrade this behaviour is fixed (swarms aren't actually used currently
and aren't consensus-relevant so recalculating early won't hurt
anything).
- Details on decomm/dereg are added to RPC info and print_sn/print_sn_status
- Slightly improves the % of reward output in the print_sn output by
rounding it to two digits, and reserves space in the output string to
avoid excessive reallocations.
- Adds various debugging at higher debug levels to quorum voting (into
all of voting itself, vote transmission, and vote reception).
- Reset service node list internal data structure version to 0. The SN
list has to be rescanned anyway at upgrade (its size has changed), so we
might as well reset the version and remove the version-dependent
serialization code. (Note that the affected code here is for SN states
in lmdb storage, not for SN-to-SN communication serialization).
2019-06-18 23:57:02 +02:00
if ( tx . type ! = txtype : : state_change )
2018-09-04 11:40:04 +02:00
return true ;
tx_verification_context tvc ;
uint64_t max_used_block_height = 0 ;
crypto : : hash max_used_block_id = null_hash ;
if ( ! m_blockchain . check_tx_inputs ( tx , max_used_block_height , max_used_block_id , tvc , /*kept_by_block*/ false ) )
{
2019-06-11 20:53:46 +02:00
LOG_PRINT_L1 ( " TX type: " < < tx . type < < " considered for relaying failed tx inputs check, txid: " < < txid < < " , reason: " < < print_tx_verification_context ( tvc , & tx ) ) ;
2018-09-04 11:40:04 +02:00
return true ;
Service Node Deregister Part 5 (#89)
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* core, service_node_list: separated address from service node pubkey
* Retrieve quorum list from height, reviewed
* Setup data structures for de/register TX
* Submit and validate partial/full deregisters
* Add P2P relaying of partial deregistration votes
* Code review adjustments for deregistration part 1
- Fix check_tx_semantic
- Remove signature_pod as votes are now stored as blobs. Serialization
overrides don't intefere with crypto::signature anymore.
* deregistration_vote_pool - changed sign/verify interface and removed repeated code
* Misc review, fix sign/verify api, vote threshold
* Deregister/tx edge case handling for combinatoric votes
* Store service node lists for the duration of deregister lifetimes
* Quorum min/max bug, sort node list, fix node to test list
* Change quorum to store acc pub address, fix oob bug
* Code review for expiring votes, acc keys to pub_key, improve err msgs
* Add early out for is_deregistration_tx and protect against quorum changes
* Remove debug code, fix segfault
* Remove irrelevant check for tx v3 in blockchain, fix >= height for pruning quorum states
Incorrect assumption that a transaction can be kept in the chain if it could
eventually become invalid, because if it were the chain would be split and
eventually these transaction would be dropped. But also that we should not
override the pre-existing logic which handles this case anyway.
2018-07-18 04:42:47 +02:00
}
}
2017-05-14 15:06:55 +02:00
txs . push_back ( std : : make_pair ( txid , bd ) ) ;
}
catch ( const std : : exception & e )
{
MERROR ( " Failed to get transaction blob from db " ) ;
// ignore error
}
2015-11-21 01:26:48 +01:00
}
}
2017-05-14 15:06:55 +02:00
return true ;
2017-11-08 13:06:41 +01:00
} , false ) ;
2015-11-21 01:26:48 +01:00
return true ;
}
//---------------------------------------------------------------------------------
2019-11-06 07:28:33 +01:00
int tx_memory_pool : : set_relayable ( const std : : vector < crypto : : hash > & tx_hashes ) {
int updated = 0 ;
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2019-11-06 07:28:33 +01:00
LockedTXN lock ( m_blockchain ) ;
for ( auto & tx : tx_hashes )
{
try {
txpool_tx_meta_t meta ;
if ( m_blockchain . get_txpool_tx_meta ( tx , meta ) & & meta . do_not_relay )
{
meta . do_not_relay = false ;
m_blockchain . update_txpool_tx ( tx , meta ) ;
+ + updated ;
}
} catch ( const std : : exception & e ) {
MERROR ( " Failed to upate txpool transaction metadata: " < < e . what ( ) ) ;
}
}
lock . commit ( ) ;
return updated ;
}
//---------------------------------------------------------------------------------
2018-04-16 01:16:02 +02:00
void tx_memory_pool : : set_relayed ( const std : : vector < std : : pair < crypto : : hash , cryptonote : : blobdata > > & txs )
2015-11-21 01:26:48 +01:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2015-11-21 01:26:48 +01:00
const time_t now = time ( NULL ) ;
2017-05-14 15:06:55 +02:00
LockedTXN lock ( m_blockchain ) ;
2019-11-06 07:28:33 +01:00
for ( auto & tx : txs )
2015-11-21 01:26:48 +01:00
{
2017-05-14 15:06:55 +02:00
try
2016-10-22 21:46:19 +02:00
{
2017-12-14 18:09:30 +01:00
txpool_tx_meta_t meta ;
2019-11-06 07:28:33 +01:00
if ( m_blockchain . get_txpool_tx_meta ( tx . first , meta ) )
2017-12-14 18:09:30 +01:00
{
meta . relayed = true ;
meta . last_relayed_time = now ;
2019-11-06 07:28:33 +01:00
m_blockchain . update_txpool_tx ( tx . first , meta ) ;
2017-12-14 18:09:30 +01:00
}
2017-05-14 15:06:55 +02:00
}
catch ( const std : : exception & e )
{
MERROR ( " Failed to update txpool transaction metadata: " < < e . what ( ) ) ;
// continue
2016-10-22 21:46:19 +02:00
}
2015-11-21 01:26:48 +01:00
}
2019-06-26 06:00:05 +02:00
lock . commit ( ) ;
2015-11-21 01:26:48 +01:00
}
//---------------------------------------------------------------------------------
2017-11-08 13:06:41 +01:00
size_t tx_memory_pool : : get_transactions_count ( bool include_unrelayed_txes ) const
2014-03-03 23:07:58 +01:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2017-11-08 13:06:41 +01:00
return m_blockchain . get_txpool_tx_count ( include_unrelayed_txes ) ;
2014-03-03 23:07:58 +01:00
}
//---------------------------------------------------------------------------------
2018-04-16 01:16:02 +02:00
void tx_memory_pool : : get_transactions ( std : : vector < transaction > & txs , bool include_unrelayed_txes ) const
2014-03-03 23:07:58 +01:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2018-04-16 01:16:02 +02:00
txs . reserve ( m_blockchain . get_txpool_tx_count ( include_unrelayed_txes ) ) ;
2017-05-14 15:06:55 +02:00
m_blockchain . for_all_txpool_txes ( [ & txs ] ( const crypto : : hash & txid , const txpool_tx_meta_t & meta , const cryptonote : : blobdata * bd ) {
transaction tx ;
if ( ! parse_and_validate_tx_from_blob ( * bd , tx ) )
{
MERROR ( " Failed to parse tx from txpool " ) ;
// continue
return true ;
}
2018-12-05 21:34:35 +01:00
tx . set_hash ( txid ) ;
txs . push_back ( std : : move ( tx ) ) ;
2017-05-14 15:06:55 +02:00
return true ;
2017-11-08 13:06:41 +01:00
} , true , include_unrelayed_txes ) ;
2014-03-03 23:07:58 +01:00
}
2015-04-23 14:13:07 +02:00
//------------------------------------------------------------------
2020-10-01 10:08:19 +02:00
void tx_memory_pool : : get_transaction_hashes ( std : : vector < crypto : : hash > & txs , bool include_unrelayed_txes , bool include_only_blinked ) const
2017-03-22 19:03:23 +01:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2018-04-16 01:16:02 +02:00
txs . reserve ( m_blockchain . get_txpool_tx_count ( include_unrelayed_txes ) ) ;
2020-10-01 10:08:19 +02:00
m_blockchain . for_all_txpool_txes ( [ & txs , include_only_blinked , this ] ( const crypto : : hash & txid , const txpool_tx_meta_t & meta , const cryptonote : : blobdata * bd ) {
bool include_tx = true ;
if ( include_only_blinked ) include_tx = has_blink ( txid ) ;
if ( include_tx ) txs . push_back ( txid ) ;
2017-05-14 15:06:55 +02:00
return true ;
2017-11-08 13:06:41 +01:00
} , false , include_unrelayed_txes ) ;
2017-03-22 19:03:23 +01:00
}
//------------------------------------------------------------------
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
void tx_memory_pool : : get_transaction_backlog ( std : : vector < rpc : : tx_backlog_entry > & backlog , bool include_unrelayed_txes ) const
2017-08-26 17:23:31 +02:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2017-08-26 17:23:31 +02:00
const uint64_t now = time ( NULL ) ;
2018-04-16 01:16:02 +02:00
backlog . reserve ( m_blockchain . get_txpool_tx_count ( include_unrelayed_txes ) ) ;
2017-08-26 17:23:31 +02:00
m_blockchain . for_all_txpool_txes ( [ & backlog , now ] ( const crypto : : hash & txid , const txpool_tx_meta_t & meta , const cryptonote : : blobdata * bd ) {
2018-07-18 23:24:53 +02:00
backlog . push_back ( { meta . weight , meta . fee , meta . receive_time - now } ) ;
2017-08-26 17:23:31 +02:00
return true ;
2017-11-08 13:06:41 +01:00
} , false , include_unrelayed_txes ) ;
2017-08-26 17:23:31 +02:00
}
//------------------------------------------------------------------
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
void tx_memory_pool : : get_transaction_stats ( struct rpc : : txpool_stats & stats , bool include_unrelayed_txes ) const
2017-05-31 20:11:56 +02:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2017-05-31 20:11:56 +02:00
const uint64_t now = time ( NULL ) ;
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
std : : map < uint64_t , rpc : : txpool_histo > agebytes ;
2017-11-08 13:06:41 +01:00
stats . txs_total = m_blockchain . get_txpool_tx_count ( include_unrelayed_txes ) ;
2018-07-18 23:24:53 +02:00
std : : vector < uint32_t > weights ;
weights . reserve ( stats . txs_total ) ;
m_blockchain . for_all_txpool_txes ( [ & stats , & weights , now , & agebytes ] ( const crypto : : hash & txid , const txpool_tx_meta_t & meta , const cryptonote : : blobdata * bd ) {
weights . push_back ( meta . weight ) ;
stats . bytes_total + = meta . weight ;
if ( ! stats . bytes_min | | meta . weight < stats . bytes_min )
stats . bytes_min = meta . weight ;
if ( meta . weight > stats . bytes_max )
stats . bytes_max = meta . weight ;
2017-05-31 20:11:56 +02:00
if ( ! meta . relayed )
stats . num_not_relayed + + ;
stats . fee_total + = meta . fee ;
if ( ! stats . oldest | | meta . receive_time < stats . oldest )
stats . oldest = meta . receive_time ;
if ( meta . receive_time < now - 600 )
stats . num_10m + + ;
if ( meta . last_failed_height )
stats . num_failing + + ;
2017-08-23 15:19:38 +02:00
uint64_t age = now - meta . receive_time + ( now = = meta . receive_time ) ;
2017-06-02 04:30:26 +02:00
agebytes [ age ] . txs + + ;
2018-07-18 23:24:53 +02:00
agebytes [ age ] . bytes + = meta . weight ;
2017-09-22 14:57:20 +02:00
if ( meta . double_spend_seen )
+ + stats . num_double_spends ;
2017-05-31 20:11:56 +02:00
return true ;
2019-08-28 16:46:31 +02:00
} , false , include_unrelayed_txes ) ;
2022-04-15 04:34:09 +02:00
stats . bytes_med = tools : : median ( std : : move ( weights ) ) ;
2017-06-02 04:30:26 +02:00
if ( stats . txs_total > 1 )
{
/* looking for 98th percentile */
size_t end = stats . txs_total * 0.02 ;
uint64_t delta , factor ;
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
std : : map < uint64_t , rpc : : txpool_histo > : : iterator it , i2 ;
2017-06-02 04:30:26 +02:00
if ( end )
{
/* If enough txs, spread the first 98% of results across
* the first 9 bins , drop final 2 % in last bin .
*/
2019-08-28 16:46:31 +02:00
it = agebytes . end ( ) ;
size_t cumulative_num = 0 ;
/* Since agebytes is not empty and end is nonzero, the
* below loop can always run at least once .
*/
do {
- - it ;
cumulative_num + = it - > second . txs ;
} while ( it ! = agebytes . begin ( ) & & cumulative_num < end ) ;
2017-06-02 04:30:26 +02:00
stats . histo_98pc = it - > first ;
factor = 9 ;
delta = it - > first ;
stats . histo . resize ( 10 ) ;
} else
{
/* If not enough txs, don't reserve the last slot;
* spread evenly across all 10 bins .
*/
stats . histo_98pc = 0 ;
it = agebytes . end ( ) ;
factor = stats . txs_total > 9 ? 10 : stats . txs_total ;
delta = now - stats . oldest ;
stats . histo . resize ( factor ) ;
}
if ( ! delta )
delta = 1 ;
for ( i2 = agebytes . begin ( ) ; i2 ! = it ; i2 + + )
{
size_t i = ( i2 - > first * factor - 1 ) / delta ;
stats . histo [ i ] . txs + = i2 - > second . txs ;
stats . histo [ i ] . bytes + = i2 - > second . bytes ;
}
for ( ; i2 ! = agebytes . end ( ) ; i2 + + )
{
stats . histo [ factor ] . txs + = i2 - > second . txs ;
stats . histo [ factor ] . bytes + = i2 - > second . bytes ;
}
}
2017-05-31 20:11:56 +02:00
}
//------------------------------------------------------------------
2016-03-25 01:03:02 +01:00
//TODO: investigate whether boolean return is appropriate
2020-08-20 06:16:37 +02:00
bool tx_memory_pool : : get_transactions_and_spent_keys_info ( std : : vector < rpc : : tx_info > & tx_infos , std : : vector < rpc : : spent_key_image_info > & key_image_infos , std : : function < void ( const transaction & , rpc : : tx_info & ) > post_process , bool include_sensitive_data ) const
2015-04-23 14:13:07 +02:00
{
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
std : : unique_lock tx_lock { m_transactions_lock , std : : defer_lock } ;
std : : unique_lock bc_lock { m_blockchain , std : : defer_lock } ;
2020-01-18 18:56:31 +01:00
auto blink_lock = blink_shared_lock ( std : : defer_lock ) ;
2020-06-02 01:01:24 +02:00
std : : lock ( tx_lock , bc_lock , blink_lock ) ;
2019-12-12 04:11:07 +01:00
2018-04-16 01:16:02 +02:00
tx_infos . reserve ( m_blockchain . get_txpool_tx_count ( ) ) ;
key_image_infos . reserve ( m_blockchain . get_txpool_tx_count ( ) ) ;
2019-11-20 01:24:48 +01:00
2020-08-20 06:16:37 +02:00
m_blockchain . for_all_txpool_txes ( [ & tx_infos , this , include_sensitive_data , post_process = std : : move ( post_process ) ] ( const crypto : : hash & txid , const txpool_tx_meta_t & meta , const cryptonote : : blobdata * bd ) {
2017-05-14 15:06:55 +02:00
transaction tx ;
if ( ! parse_and_validate_tx_from_blob ( * bd , tx ) )
{
MERROR ( " Failed to parse tx from txpool " ) ;
// continue
return true ;
}
2019-11-20 01:24:48 +01:00
tx_infos . emplace_back ( ) ;
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
auto & txi = tx_infos . back ( ) ;
2020-10-23 22:32:28 +02:00
txi . id_hash = tools : : type_to_hex ( txid ) ;
2019-11-20 01:24:48 +01:00
txi . tx_blob = * bd ;
2018-12-05 21:34:10 +01:00
tx . set_hash ( txid ) ;
2017-05-14 15:06:55 +02:00
txi . tx_json = obj_to_json_str ( tx ) ;
2018-07-18 23:24:53 +02:00
txi . blob_size = bd - > size ( ) ;
txi . weight = meta . weight ;
2017-05-14 15:06:55 +02:00
txi . fee = meta . fee ;
txi . kept_by_block = meta . kept_by_block ;
txi . max_used_block_height = meta . max_used_block_height ;
2020-10-23 22:32:28 +02:00
txi . max_used_block_id_hash = tools : : type_to_hex ( meta . max_used_block_id ) ;
2017-05-14 15:06:55 +02:00
txi . last_failed_height = meta . last_failed_height ;
2020-10-23 22:32:28 +02:00
txi . last_failed_id_hash = tools : : type_to_hex ( meta . last_failed_id ) ;
2017-11-08 13:06:41 +01:00
// In restricted mode we do not include this data:
txi . receive_time = include_sensitive_data ? meta . receive_time : 0 ;
2017-05-14 15:06:55 +02:00
txi . relayed = meta . relayed ;
2017-11-08 13:06:41 +01:00
// In restricted mode we do not include this data:
txi . last_relayed_time = include_sensitive_data ? meta . last_relayed_time : 0 ;
2017-05-14 15:06:55 +02:00
txi . do_not_relay = meta . do_not_relay ;
2017-09-22 14:57:20 +02:00
txi . double_spend_seen = meta . double_spend_seen ;
Fix rare blink pool deadlock
cryptonote_protocol_handler calls `pool.get_blink(hash)` while already
holding a blink shared lock, which should have been
`pool.get_blink(hash, true)` to avoid `get_blink` trying to take its own
lock.
That double lock is undefined behaviour and can cause a deadlock on the
mutex, although it appears rare that it actually does. If it does,
however, this eventually backs up into vote relaying during the idle
loop, which then stalls the idle loop so we stop sending out uptime
proofs (since that is also in the idle loop).
A simple fix here is to add the `true` argument, but on reconsideration
this extra argument to take or not take a lock is messy and error prone,
so this commit instead removes the second argument entirely and instead
documents which call must and must not hold a lock, getting rid of the
three methods (get_blink, has_blink, and add_existing_blink) that had
the `have_lock` argument. This ends up having only a small impact on
calling code - the vast majority of callers already hold a lock, and the
few that don't are easily adjusted.
2020-01-18 19:01:45 +01:00
txi . blink = has_blink ( txid ) ;
2020-08-20 06:16:37 +02:00
if ( post_process )
post_process ( tx , txi ) ;
2017-05-14 15:06:55 +02:00
return true ;
2017-11-08 13:06:41 +01:00
} , true , include_sensitive_data ) ;
2015-04-23 14:13:07 +02:00
2017-11-08 13:06:41 +01:00
txpool_tx_meta_t meta ;
2015-04-23 14:13:07 +02:00
for ( const key_images_container : : value_type & kee : m_spent_key_images ) {
const crypto : : key_image & k_image = kee . first ;
const std : : unordered_set < crypto : : hash > & kei_image_set = kee . second ;
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
rpc : : spent_key_image_info ki { } ;
2020-10-23 22:32:28 +02:00
ki . id_hash = tools : : type_to_hex ( k_image ) ;
2015-04-23 14:13:07 +02:00
for ( const crypto : : hash & tx_id_hash : kei_image_set )
{
2017-11-08 13:06:41 +01:00
if ( ! include_sensitive_data )
{
try
{
2017-12-14 18:09:30 +01:00
if ( ! m_blockchain . get_txpool_tx_meta ( tx_id_hash , meta ) )
{
MERROR ( " Failed to get tx meta from txpool " ) ;
return false ;
}
2017-11-08 13:06:41 +01:00
if ( ! meta . relayed )
// Do not include that transaction if in restricted mode and it's not relayed
continue ;
}
catch ( const std : : exception & e )
{
MERROR ( " Failed to get tx meta from txpool: " < < e . what ( ) ) ;
return false ;
}
}
2020-10-23 22:32:28 +02:00
ki . txs_hashes . push_back ( tools : : type_to_hex ( tx_id_hash ) ) ;
2015-04-23 14:13:07 +02:00
}
2017-11-08 13:06:41 +01:00
// Only return key images for which we have at least one tx that we can show for them
if ( ! ki . txs_hashes . empty ( ) )
key_image_infos . push_back ( ki ) ;
2015-04-23 14:13:07 +02:00
}
return true ;
}
2014-03-03 23:07:58 +01:00
//---------------------------------------------------------------------------------
2019-10-06 05:28:30 +02:00
bool tx_memory_pool : : check_for_key_images ( const std : : vector < crypto : : key_image > & key_images , std : : vector < bool > & spent ) const
2017-09-05 18:20:27 +02:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2017-09-05 18:20:27 +02:00
spent . clear ( ) ;
for ( const auto & image : key_images )
{
spent . push_back ( m_spent_key_images . find ( image ) = = m_spent_key_images . end ( ) ? false : true ) ;
}
return true ;
}
//---------------------------------------------------------------------------------
2019-11-20 01:24:48 +01:00
int tx_memory_pool : : find_transactions ( const std : : vector < crypto : : hash > & tx_hashes , std : : vector < cryptonote : : blobdata > & txblobs ) const
2014-03-03 23:07:58 +01:00
{
2019-11-20 01:24:48 +01:00
if ( tx_hashes . empty ( ) )
return 0 ;
txblobs . reserve ( txblobs . size ( ) + tx_hashes . size ( ) ) ;
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2019-11-20 01:24:48 +01:00
int added = 0 ;
for ( auto & id : tx_hashes )
2017-05-14 15:06:55 +02:00
{
2019-11-20 01:24:48 +01:00
try
{
cryptonote : : blobdata txblob ;
m_blockchain . get_txpool_tx_blob ( id , txblob ) ;
txblobs . push_back ( std : : move ( txblob ) ) ;
+ + added ;
}
catch ( . . . ) { /* ignore */ }
2017-05-14 15:06:55 +02:00
}
2019-11-20 01:24:48 +01:00
return added ;
}
//---------------------------------------------------------------------------------
bool tx_memory_pool : : get_transaction ( const crypto : : hash & id , cryptonote : : blobdata & txblob ) const
{
std : : vector < cryptonote : : blobdata > found ;
find_transactions ( { { id } } , found ) ;
if ( found . empty ( ) )
2014-03-03 23:07:58 +01:00
return false ;
2019-11-20 01:24:48 +01:00
txblob = std : : move ( found [ 0 ] ) ;
return true ;
2014-03-03 23:07:58 +01:00
}
//---------------------------------------------------------------------------------
2019-10-27 23:26:38 +01:00
bool tx_memory_pool : : on_blockchain_inc ( block const & blk )
2014-03-03 23:07:58 +01:00
{
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
std : : unique_lock lock { m_transactions_lock } ;
2018-06-24 11:46:57 +02:00
m_input_cache . clear ( ) ;
2018-10-31 14:52:16 +01:00
m_parsed_tx_cache . clear ( ) ;
2019-08-12 01:48:47 +02:00
std : : vector < transaction > pool_txs ;
get_transactions ( pool_txs ) ;
if ( pool_txs . empty ( ) ) return true ;
// NOTE: For transactions in the pool, on new block received, if a Service
// Node changed state any older state changes that the node cannot
// transition to now are invalid and cannot be used, so take them out from
// the pool.
// Otherwise multiple state changes can queue up until they are applicable
// and be applied on the node.
2019-08-15 08:04:43 +02:00
uint64_t const block_height = cryptonote : : get_block_height ( blk ) ;
2019-10-27 23:26:38 +01:00
auto & service_node_list = m_blockchain . get_service_node_list ( ) ;
2019-08-12 01:48:47 +02:00
for ( transaction const & pool_tx : pool_txs )
{
tx_extra_service_node_state_change state_change ;
crypto : : public_key service_node_pubkey ;
if ( pool_tx . type = = txtype : : state_change & &
get_service_node_state_change_from_tx_extra ( pool_tx . extra , state_change , blk . major_version ) )
{
2021-01-04 01:09:45 +01:00
// TODO(oxen): PERF(oxen): On pop_blocks we return all the TXs to the
2019-08-15 08:04:43 +02:00
// pool. The greater the pop_blocks, the more txs that are queued in the
// pool, and for every subsequent block you sync, get_transactions has
// to allocate these transactions and we have to search every
// transaction in the pool every synced block- causing great slowdown.
// It'd be nice to optimise this or rearchitect the way this pruning is
// done to be smarter.
if ( state_change . block_height > = block_height ) // NOTE: Can occur if we pop_blocks and old popped state changes are returned to the pool.
continue ;
2019-08-12 01:48:47 +02:00
if ( service_node_list . get_quorum_pubkey ( service_nodes : : quorum_type : : obligations ,
service_nodes : : quorum_group : : worker ,
state_change . block_height ,
state_change . service_node_index ,
service_node_pubkey ) )
{
2019-08-12 08:40:04 +02:00
crypto : : hash tx_hash ;
if ( ! get_transaction_hash ( pool_tx , tx_hash ) )
{
MERROR ( " Failed to get transaction hash from txpool to check if we can prune a state change " ) ;
continue ;
}
txpool_tx_meta_t meta ;
if ( ! m_blockchain . get_txpool_tx_meta ( tx_hash , meta ) )
{
MERROR ( " Failed to get tx meta from txpool to check if we can prune a state change " ) ;
continue ;
}
if ( meta . kept_by_block ) // Do not prune transaction if kept by block (belongs to alt block, so we need incase we switch to alt-chain)
continue ;
2019-08-12 01:48:47 +02:00
std : : vector < service_nodes : : service_node_pubkey_info > service_node_array = service_node_list . get_service_node_list_state ( { service_node_pubkey } ) ;
if ( service_node_array . empty ( ) | |
2019-10-30 04:50:34 +01:00
! service_node_array [ 0 ] . info - > can_transition_to_state ( blk . major_version , state_change . block_height , state_change . state ) )
2019-08-12 01:48:47 +02:00
{
transaction tx ;
cryptonote : : blobdata blob ;
size_t tx_weight ;
uint64_t fee ;
bool relayed , do_not_relay , double_spend_seen ;
2019-08-12 08:40:04 +02:00
take_tx ( tx_hash , tx , blob , tx_weight , fee , relayed , do_not_relay , double_spend_seen ) ;
2019-08-12 01:48:47 +02:00
}
}
}
}
2014-03-03 23:07:58 +01:00
return true ;
}
//---------------------------------------------------------------------------------
2019-08-12 01:48:47 +02:00
bool tx_memory_pool : : on_blockchain_dec ( )
2014-03-03 23:07:58 +01:00
{
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
std : : unique_lock lock { m_transactions_lock } ;
2018-06-24 11:46:57 +02:00
m_input_cache . clear ( ) ;
2018-10-31 14:52:16 +01:00
m_parsed_tx_cache . clear ( ) ;
2014-03-03 23:07:58 +01:00
return true ;
}
2019-11-06 07:28:33 +01:00
//------------------------------------------------------------------
std : : vector < uint8_t > tx_memory_pool : : have_txs ( const std : : vector < crypto : : hash > & hashes ) const
2014-03-03 23:07:58 +01:00
{
2019-11-06 07:28:33 +01:00
std : : vector < uint8_t > result ( hashes . size ( ) , false ) ;
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2019-11-06 07:28:33 +01:00
auto & db = m_blockchain . get_db ( ) ;
for ( size_t i = 0 ; i < hashes . size ( ) ; i + + )
result [ i ] = db . txpool_has_tx ( hashes [ i ] ) ;
return result ;
}
//---------------------------------------------------------------------------------
bool tx_memory_pool : : have_tx ( const crypto : : hash & id ) const
{
return have_txs ( { { id } } ) [ 0 ] ;
2014-03-03 23:07:58 +01:00
}
//---------------------------------------------------------------------------------
2019-11-25 05:57:15 +01:00
bool tx_memory_pool : : have_tx_keyimges_as_spent ( const transaction & tx , std : : vector < crypto : : hash > * conflicting ) const
2014-03-03 23:07:58 +01:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2021-02-08 06:31:05 +01:00
2019-11-25 05:57:15 +01:00
bool ret = false ;
2017-01-22 21:38:10 +01:00
for ( const auto & in : tx . vin )
2014-03-03 23:07:58 +01:00
{
2020-06-02 05:47:20 +02:00
CHECKED_GET_SPECIFIC_VARIANT ( in , txin_to_key , tokey_in , true ) ; //should never fail
2019-11-25 05:57:15 +01:00
auto it = m_spent_key_images . find ( tokey_in . k_image ) ;
if ( it ! = m_spent_key_images . end ( ) )
{
if ( ! conflicting )
return true ;
ret = true ;
conflicting - > insert ( conflicting - > end ( ) , it - > second . begin ( ) , it - > second . end ( ) ) ;
}
2014-03-03 23:07:58 +01:00
}
2019-11-25 05:57:15 +01:00
return ret ;
2014-03-03 23:07:58 +01:00
}
//---------------------------------------------------------------------------------
2014-07-17 16:31:44 +02:00
bool tx_memory_pool : : have_tx_keyimg_as_spent ( const crypto : : key_image & key_im ) const
2014-03-03 23:07:58 +01:00
{
C++17
Switch loki dev branch to C++17 compilation, and update the code with
various C++17 niceties.
- stop including the (deprecated) lokimq/string_view.h header and
instead switch everything to use std::string_view and `""sv` instead of
`""_sv`.
- std::string_view is much nicer than epee::span, so updated various
loki-specific code to use it instead.
- made epee "portable storage" serialization accept a std::string_view
instead of const lvalue std::string so that we can avoid copying.
- switched from mapbox::variant to std::variant
- use `auto [a, b] = whatever()` instead of `T1 a; T2 b; std::tie(a, b)
= whatever()` in a couple places (in the wallet code).
- switch to std::lock(...) instead of boost::lock(...) for simultaneous
lock acquisition. boost::lock() won't compile in C++17 mode when given
locks of different types.
- removed various pre-C++17 workarounds, e.g. for fold expressions,
unused argument attributes, and byte-spannable object detection.
- class template deduction means lock types no longer have to specify
the mutex, so `std::unique_lock<std::mutex> lock{mutex}` can become
`std::unique_lock lock{mutex}`. This will make switching any mutex
types (e.g. from boost to std mutexes) far easier as you just have to
update the type in the header and everything should work. This also
makes the tools::unique_lock and tools::shared_lock methods redundant
(which were a sort of poor-mans-pre-C++17 way to eliminate the
redundancy) so they are now gone and replaced with direct unique_lock or
shared_lock constructions.
- Redid the LNS validation using a string_view; instead of using raw
char pointers the code now uses a string view and chops off parts of the
view as it validates. So, for instance, it starts with "abcd.loki",
validates the ".loki" and chops the view to "abcd", then validates the
first character and chops to "bcd", validates the last and chops to
"bc", then can just check everything remaining for is-valid-middle-char.
- LNS validation gained a couple minor validation checks in the process:
- slightly tightened the requirement on lokinet addresses to require
that the last character of the mapped address is 'y' or 'o' (the
last base32z char holds only one significant bit).
- In parse_owner_to_generic_owner made sure that the owner value has
the correct size (otherwise we could up end not filling or
overfilling the pubkey buffer).
- Replaced base32z/base64/hex conversions with lokimq's versions which
have a nicer interface, are better optimized, and don't depend on epee.
2020-05-13 20:12:49 +02:00
std : : unique_lock lock { m_transactions_lock } ;
2014-03-03 23:07:58 +01:00
return m_spent_key_images . end ( ) ! = m_spent_key_images . find ( key_im ) ;
}
//---------------------------------------------------------------------------------
2019-12-26 07:38:00 +01:00
bool tx_memory_pool : : check_tx_inputs ( const std : : function < cryptonote : : transaction & ( ) > & get_tx , const crypto : : hash & txid , uint64_t & max_used_block_height ,
crypto : : hash & max_used_block_id , tx_verification_context & tvc , bool kept_by_block , uint64_t * blink_rollback_height ) const
2018-06-24 11:46:57 +02:00
{
if ( ! kept_by_block )
{
const std : : unordered_map < crypto : : hash , std : : tuple < bool , tx_verification_context , uint64_t , crypto : : hash > > : : const_iterator i = m_input_cache . find ( txid ) ;
if ( i ! = m_input_cache . end ( ) )
{
max_used_block_height = std : : get < 2 > ( i - > second ) ;
max_used_block_id = std : : get < 3 > ( i - > second ) ;
tvc = std : : get < 1 > ( i - > second ) ;
return std : : get < 0 > ( i - > second ) ;
}
}
2019-12-26 07:38:00 +01:00
std : : unordered_set < crypto : : key_image > key_image_conflicts ;
bool ret = m_blockchain . check_tx_inputs ( get_tx ( ) , max_used_block_height , max_used_block_id , tvc , kept_by_block , blink_rollback_height ? & key_image_conflicts : nullptr ) ;
if ( ret & & ! key_image_conflicts . empty ( ) )
{
// There are some key image conflicts, but since we have blink_rollback_height this is an
// approved blink tx that we want to accept via rollback, if possible.
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
uint64_t immutable = m_blockchain . get_immutable_height ( ) ;
uint64_t height = m_blockchain . get_current_blockchain_height ( ) ;
bool can_fix_with_a_rollback = false ;
if ( height - immutable > 100 )
{
// Sanity check; if this happens checkpoints are failing and we can't guarantee blinks
// anyway (because the blink quorums are not immutable).
MERROR ( " Unable to scan for conflicts: blockchain checkpoints are too far back " ) ;
}
else
{
MDEBUG ( " Found " < < key_image_conflicts . size ( ) < < " conflicting key images for blink tx " < < txid < < " ; checking to see if we can roll back " ) ;
// Check all the key images of all the blockchain transactions in blocks since the immutable
// height, and remove any conflicts from the set of conflicts, updating the rollback height
// as we go. If we remove all then rolling back will work, and we can accept the blink,
// otherwise we have to refuse it (because immutable blocks have to trump a blink tx).
//
// This sounds expensive, but in reality the blocks since the immutable checkpoint is
// usually only around 8-12, we do this in reverse order (conflicts are most likely to be in
// the last block or two), and there is little incentive to actively exploit this since this
// code is here, and even if someone did want to they'd have to also be 51% attacking the
// network to wipe out recently mined blinks -- but that can't work anyway.
//
2020-09-02 06:21:44 +02:00
std : : vector < cryptonote : : block > blocks ;
if ( m_blockchain . get_blocks_only ( immutable + 1 , height , blocks ) )
2019-12-26 07:38:00 +01:00
{
std : : vector < cryptonote : : transaction > txs ;
std : : vector < crypto : : hash > missed_txs ;
uint64_t earliest = height ;
for ( auto it = blocks . rbegin ( ) ; it ! = blocks . rend ( ) ; it + + )
{
2020-09-02 06:21:44 +02:00
const auto & block = * it ;
2019-12-26 07:38:00 +01:00
auto block_height = cryptonote : : get_block_height ( block ) ;
txs . clear ( ) ;
missed_txs . clear ( ) ;
if ( ! m_blockchain . get_transactions ( block . tx_hashes , txs , missed_txs ) )
{
MERROR ( " Unable to get transactions for block " < < block . hash ) ;
can_fix_with_a_rollback = false ;
break ;
}
for ( const auto & tx : txs ) {
for ( const auto & in : tx . vin ) {
2020-10-16 00:44:36 +02:00
if ( auto * ttk = std : : get_if < txin_to_key > ( & in ) ; ttk & & key_image_conflicts . erase ( ttk - > k_image ) ) {
2019-12-26 07:38:00 +01:00
earliest = std : : min ( earliest , block_height ) ;
if ( key_image_conflicts . empty ( ) )
goto end ;
}
}
}
}
end :
if ( key_image_conflicts . empty ( ) & & earliest < height & & earliest > immutable )
{
MDEBUG ( " Blink admission requires rolling back to height " < < earliest ) ;
can_fix_with_a_rollback = true ;
if ( * blink_rollback_height = = 0 | | * blink_rollback_height > earliest )
* blink_rollback_height = earliest ;
}
}
else
MERROR ( " Failed to retrieve blocks for trying a blink rollback! " ) ;
}
if ( ! can_fix_with_a_rollback )
{
MWARNING ( " Blink admission of " < < txid < < " is not possible even with a rollback: found " < < key_image_conflicts . size ( ) < < " key image conflicts in immutable blocks " ) ;
ret = false ;
tvc . m_double_spend = true ;
}
}
2018-06-24 11:46:57 +02:00
if ( ! kept_by_block )
m_input_cache . insert ( std : : make_pair ( txid , std : : make_tuple ( ret , tvc , max_used_block_height , max_used_block_id ) ) ) ;
return ret ;
}
//---------------------------------------------------------------------------------
bool tx_memory_pool : : is_transaction_ready_to_go ( txpool_tx_meta_t & txd , const crypto : : hash & txid , const cryptonote : : blobdata & txblob , transaction & tx ) const
2014-03-03 23:07:58 +01:00
{
2018-06-01 22:50:46 +02:00
struct transction_parser
{
2018-12-05 21:34:10 +01:00
transction_parser ( const cryptonote : : blobdata & txblob , const crypto : : hash & txid , transaction & tx ) : txblob ( txblob ) , txid ( txid ) , tx ( tx ) , parsed ( false ) { }
2018-06-01 22:50:46 +02:00
cryptonote : : transaction & operator ( ) ( )
{
if ( ! parsed )
{
if ( ! parse_and_validate_tx_from_blob ( txblob , tx ) )
throw std : : runtime_error ( " failed to parse transaction blob " ) ;
2018-12-05 21:34:10 +01:00
tx . set_hash ( txid ) ;
2018-06-01 22:50:46 +02:00
parsed = true ;
}
return tx ;
}
const cryptonote : : blobdata & txblob ;
2018-12-05 21:34:10 +01:00
const crypto : : hash & txid ;
2018-06-01 22:50:46 +02:00
transaction & tx ;
bool parsed ;
2018-12-05 21:34:10 +01:00
} lazy_tx ( txblob , txid , tx ) ;
2018-06-01 22:50:46 +02:00
2014-03-03 23:07:58 +01:00
//not the best implementation at this time, sorry :(
//check is ring_signature already checked ?
if ( txd . max_used_block_id = = null_hash )
{ //not checked, lets try to check
if ( txd . last_failed_id ! = null_hash & & m_blockchain . get_current_blockchain_height ( ) > txd . last_failed_height & & txd . last_failed_id = = m_blockchain . get_block_id_by_height ( txd . last_failed_height ) )
return false ; //we already sure that this tx is broken for this height
2016-03-27 13:35:36 +02:00
tx_verification_context tvc ;
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
if ( ! check_tx_inputs ( lazy_tx , txid , txd . max_used_block_height , txd . max_used_block_id , tvc ) )
2014-03-03 23:07:58 +01:00
{
txd . last_failed_height = m_blockchain . get_current_blockchain_height ( ) - 1 ;
txd . last_failed_id = m_blockchain . get_block_id_by_height ( txd . last_failed_height ) ;
return false ;
}
} else
{
if ( txd . max_used_block_height > = m_blockchain . get_current_blockchain_height ( ) )
return false ;
2016-04-05 14:06:29 +02:00
if ( true )
2014-03-03 23:07:58 +01:00
{
//if we already failed on this height and id, skip actual ring signature check
if ( txd . last_failed_id = = m_blockchain . get_block_id_by_height ( txd . last_failed_height ) )
return false ;
//check ring signature again, it is possible (with very small chance) that this transaction become again valid
2016-03-27 13:35:36 +02:00
tx_verification_context tvc ;
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
if ( ! check_tx_inputs ( lazy_tx , txid , txd . max_used_block_height , txd . max_used_block_id , tvc ) )
2014-03-03 23:07:58 +01:00
{
txd . last_failed_height = m_blockchain . get_current_blockchain_height ( ) - 1 ;
txd . last_failed_id = m_blockchain . get_block_id_by_height ( txd . last_failed_height ) ;
return false ;
}
}
}
//if we here, transaction seems valid, but, anyway, check for key_images collisions with blockchain, just to be sure
2018-06-01 22:50:46 +02:00
if ( m_blockchain . have_tx_keyimges_as_spent ( lazy_tx ( ) ) )
2017-09-22 14:57:20 +02:00
{
txd . double_spend_seen = true ;
2014-03-03 23:07:58 +01:00
return false ;
2017-09-22 14:57:20 +02:00
}
2014-03-03 23:07:58 +01:00
//transaction is ok.
return true ;
}
//---------------------------------------------------------------------------------
2020-09-25 02:09:54 +02:00
/**
* @ brief check if any of a transaction ' s spent key images are present in a given set
*
* @ param kic the set of key images to check against
* @ param tx the transaction to check
*
* @ return true if any key images present in the set , otherwise false
*/
static bool have_key_images ( const std : : unordered_set < crypto : : key_image > & k_images , const transaction_prefix & tx )
2014-03-03 23:07:58 +01:00
{
for ( size_t i = 0 ; i ! = tx . vin . size ( ) ; i + + )
{
2020-06-02 05:47:20 +02:00
CHECKED_GET_SPECIFIC_VARIANT ( tx . vin [ i ] , txin_to_key , itk , false ) ;
2014-03-03 23:07:58 +01:00
if ( k_images . count ( itk . k_image ) )
return true ;
}
return false ;
}
//---------------------------------------------------------------------------------
2020-09-25 02:09:54 +02:00
/**
* @ brief append the key images from a transaction to the given set
*
* @ param kic the set of key images to append to
* @ param tx the transaction
*
* @ return false if any append fails , otherwise true
*/
static bool append_key_images ( std : : unordered_set < crypto : : key_image > & k_images , const transaction_prefix & tx )
2014-03-03 23:07:58 +01:00
{
for ( size_t i = 0 ; i ! = tx . vin . size ( ) ; i + + )
{
2020-06-02 05:47:20 +02:00
CHECKED_GET_SPECIFIC_VARIANT ( tx . vin [ i ] , txin_to_key , itk , false ) ;
2014-03-03 23:07:58 +01:00
auto i_res = k_images . insert ( itk . k_image ) ;
CHECK_AND_ASSERT_MES ( i_res . second , false , " internal error: key images pool cache - inserted duplicate image in set: " < < itk . k_image ) ;
}
return true ;
}
//---------------------------------------------------------------------------------
2017-09-22 14:57:20 +02:00
void tx_memory_pool : : mark_double_spend ( const transaction & tx )
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2016-04-17 12:04:01 +02:00
bool changed = false ;
2017-09-22 14:57:20 +02:00
LockedTXN lock ( m_blockchain ) ;
for ( size_t i = 0 ; i ! = tx . vin . size ( ) ; i + + )
{
2020-06-02 05:47:20 +02:00
CHECKED_GET_SPECIFIC_VARIANT ( tx . vin [ i ] , txin_to_key , itk , void ( ) ) ;
2017-09-22 14:57:20 +02:00
const key_images_container : : const_iterator it = m_spent_key_images . find ( itk . k_image ) ;
if ( it ! = m_spent_key_images . end ( ) )
{
for ( const crypto : : hash & txid : it - > second )
{
2017-12-14 18:09:30 +01:00
txpool_tx_meta_t meta ;
if ( ! m_blockchain . get_txpool_tx_meta ( txid , meta ) )
{
MERROR ( " Failed to find tx meta in txpool " ) ;
// continue, not fatal
continue ;
}
2017-09-22 14:57:20 +02:00
if ( ! meta . double_spend_seen )
{
MDEBUG ( " Marking " < < txid < < " as double spending " < < itk . k_image ) ;
meta . double_spend_seen = true ;
2016-04-17 12:04:01 +02:00
changed = true ;
2017-09-22 14:57:20 +02:00
try
{
m_blockchain . update_txpool_tx ( txid , meta ) ;
}
catch ( const std : : exception & e )
{
MERROR ( " Failed to update tx meta: " < < e . what ( ) ) ;
// continue, not fatal
}
}
}
}
}
2019-06-26 06:00:05 +02:00
lock . commit ( ) ;
2016-04-17 12:04:01 +02:00
if ( changed )
+ + m_cookie ;
2017-09-22 14:57:20 +02:00
}
//---------------------------------------------------------------------------------
2016-03-25 01:03:02 +01:00
//TODO: investigate whether boolean return is appropriate
2020-09-25 02:50:45 +02:00
bool tx_memory_pool : : fill_block_template ( block & bl , size_t median_weight , uint64_t already_generated_coins , size_t & total_weight , uint64_t & raw_fee , uint64_t & expected_reward , uint8_t version , uint64_t height )
2014-04-19 22:53:40 +02:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2014-03-03 23:07:58 +01:00
2020-09-24 07:07:37 +02:00
total_weight = 0 ;
2020-09-25 02:50:45 +02:00
raw_fee = 0 ;
2020-09-24 07:07:37 +02:00
uint64_t best_reward = 0 ;
{
// NOTE: Calculate base line empty block reward
2021-01-04 01:09:45 +01:00
oxen_block_reward_context block_reward_context = { } ;
2020-09-24 07:07:37 +02:00
block_reward_context . height = height ;
2020-09-23 05:27:43 +02:00
2020-09-24 07:07:37 +02:00
block_reward_parts reward_parts = { } ;
2021-01-04 01:09:45 +01:00
if ( ! get_oxen_block_reward ( median_weight , total_weight , already_generated_coins , version , reward_parts , block_reward_context ) )
2020-09-24 07:07:37 +02:00
{
MERROR ( " Failed to get block reward for empty block " ) ;
return false ;
}
2017-01-13 00:39:24 +01:00
2020-09-24 07:07:37 +02:00
best_reward = version > = cryptonote : : network_version_16_pulse ? 0 /*Empty block, starts with 0 fee*/ : reward_parts . base_miner ;
2020-05-28 07:02:40 +02:00
}
2020-09-24 07:07:37 +02:00
2020-09-23 05:27:43 +02:00
size_t const max_total_weight = 2 * median_weight - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE ;
2014-04-02 18:00:17 +02:00
std : : unordered_set < crypto : : key_image > k_images ;
2014-05-27 01:51:22 +02:00
2018-07-18 23:24:53 +02:00
LOG_PRINT_L2 ( " Filling block template, median weight " < < median_weight < < " , " < < m_txs_by_fee_and_receive_time . size ( ) < < " txes in the pool " ) ;
2017-08-29 17:35:19 +02:00
LockedTXN lock ( m_blockchain ) ;
2020-09-24 06:51:57 +02:00
uint64_t next_reward = 0 ;
2020-09-25 02:50:45 +02:00
uint64_t net_fee = 0 ;
2020-09-23 05:27:43 +02:00
for ( auto sorted_it : m_txs_by_fee_and_receive_time )
2014-04-19 22:53:40 +02:00
{
2017-12-14 18:09:30 +01:00
txpool_tx_meta_t meta ;
2020-09-23 05:27:43 +02:00
if ( ! m_blockchain . get_txpool_tx_meta ( sorted_it . second , meta ) )
2017-12-14 18:09:30 +01:00
{
MERROR ( " failed to find tx meta " ) ;
continue ;
}
2020-09-24 06:51:57 +02:00
LOG_PRINT_L2 ( " Considering " < < sorted_it . second < < " , weight " < < meta . weight < < " , current block weight " < < total_weight < < " / " < < max_total_weight < < " , current reward " < < print_money ( best_reward ) ) ;
2015-04-30 07:02:12 +02:00
2018-07-18 23:24:53 +02:00
// Can not exceed maximum block weight
if ( max_total_weight < total_weight + meta . weight )
2015-07-11 07:09:21 +02:00
{
2018-07-18 23:24:53 +02:00
LOG_PRINT_L2 ( " would exceed maximum block weight " ) ;
2015-07-15 07:47:07 +02:00
continue ;
2015-07-11 07:09:21 +02:00
}
2014-03-03 23:07:58 +01:00
2020-09-24 07:07:37 +02:00
// NOTE: Calculate the next block reward for the block producer
2021-01-04 01:09:45 +01:00
oxen_block_reward_context next_block_reward_context = { } ;
2020-09-24 07:07:37 +02:00
next_block_reward_context . height = height ;
2020-09-25 02:50:45 +02:00
next_block_reward_context . fee = raw_fee + meta . fee ;
2020-09-24 07:07:37 +02:00
block_reward_parts next_reward_parts = { } ;
2021-01-04 01:09:45 +01:00
if ( ! get_oxen_block_reward ( median_weight , total_weight + meta . weight , already_generated_coins , version , next_reward_parts , next_block_reward_context ) )
2015-07-11 07:09:21 +02:00
{
2020-09-23 05:27:43 +02:00
LOG_PRINT_L2 ( " Block reward calculation bug " ) ;
return false ;
}
2018-11-12 04:02:21 +01:00
2020-09-25 02:50:45 +02:00
// NOTE: Use the net fee for comparison (after penalty is applied).
// After HF16, penalty is applied on the miner fee. Before, penalty is
// applied on the base reward.
2020-09-24 07:07:37 +02:00
if ( version > = cryptonote : : network_version_16_pulse )
2020-09-25 02:50:45 +02:00
{
2020-09-25 03:49:52 +02:00
next_reward = next_reward_parts . miner_fee ;
2020-09-25 02:50:45 +02:00
}
2020-09-24 07:07:37 +02:00
else
2020-09-25 02:50:45 +02:00
{
2020-09-25 03:49:52 +02:00
next_reward = next_reward_parts . base_miner + next_reward_parts . miner_fee ;
assert ( next_reward_parts . miner_fee = = raw_fee + meta . fee ) ;
2020-09-25 02:50:45 +02:00
}
2020-09-24 07:07:37 +02:00
// If we're getting lower reward tx, don't include this TX
2020-09-24 06:51:57 +02:00
if ( next_reward < best_reward )
2020-09-23 05:27:43 +02:00
{
2020-09-24 06:51:57 +02:00
LOG_PRINT_L2 ( " would decrease reward to " < < print_money ( next_reward ) ) ;
2020-09-23 05:27:43 +02:00
continue ;
2017-01-06 21:34:37 +01:00
}
2014-05-26 02:25:37 +02:00
2020-09-23 05:27:43 +02:00
cryptonote : : blobdata txblob = m_blockchain . get_txpool_tx_blob ( sorted_it . second ) ;
2017-05-14 15:06:55 +02:00
cryptonote : : transaction tx ;
2014-05-26 02:25:37 +02:00
// Skip transactions that are not ready to be
// included into the blockchain or that are
// missing key images
2017-08-23 19:21:14 +02:00
const cryptonote : : txpool_tx_meta_t original_meta = meta ;
2018-06-01 22:50:46 +02:00
bool ready = false ;
try
{
2020-09-23 05:27:43 +02:00
ready = is_transaction_ready_to_go ( meta , sorted_it . second , txblob , tx ) ;
2018-06-01 22:50:46 +02:00
}
catch ( const std : : exception & e )
{
MERROR ( " Failed to check transaction readiness: " < < e . what ( ) ) ;
// continue, not fatal
}
2017-08-23 19:21:14 +02:00
if ( memcmp ( & original_meta , & meta , sizeof ( meta ) ) )
{
try
2020-09-23 05:27:43 +02:00
{
m_blockchain . update_txpool_tx ( sorted_it . second , meta ) ;
}
2017-08-23 19:21:14 +02:00
catch ( const std : : exception & e )
2020-09-23 05:27:43 +02:00
{
MERROR ( " Failed to update tx meta: " < < e . what ( ) ) ;
// continue, not fatal
}
2017-08-23 19:21:14 +02:00
}
if ( ! ready )
2015-07-11 07:09:21 +02:00
{
2017-03-03 23:32:51 +01:00
LOG_PRINT_L2 ( " not ready to go " ) ;
continue ;
}
2017-05-14 15:06:55 +02:00
if ( have_key_images ( k_images , tx ) )
2017-03-03 23:32:51 +01:00
{
LOG_PRINT_L2 ( " key images already seen " ) ;
2015-07-15 07:47:07 +02:00
continue ;
2015-07-11 07:09:21 +02:00
}
2014-04-02 18:00:17 +02:00
2020-09-23 05:27:43 +02:00
bl . tx_hashes . push_back ( sorted_it . second ) ;
2018-07-18 23:24:53 +02:00
total_weight + = meta . weight ;
2020-09-25 02:50:45 +02:00
raw_fee + = meta . fee ;
2020-09-25 03:49:52 +02:00
net_fee = next_reward_parts . miner_fee ;
2020-09-25 02:50:45 +02:00
best_reward = next_reward ;
2017-05-14 15:06:55 +02:00
append_key_images ( k_images , tx ) ;
2020-09-24 06:51:57 +02:00
LOG_PRINT_L2 ( " added, new block weight " < < total_weight < < " / " < < max_total_weight < < " , reward " < < print_money ( best_reward ) ) ;
2014-03-03 23:07:58 +01:00
}
2019-06-26 06:00:05 +02:00
lock . commit ( ) ;
2014-03-03 23:07:58 +01:00
2020-09-24 06:51:57 +02:00
expected_reward = best_reward ;
2018-07-18 23:24:53 +02:00
LOG_PRINT_L2 ( " Block template filled with " < < bl . tx_hashes . size ( ) < < " txes, weight "
2020-09-24 06:51:57 +02:00
< < total_weight < < " / " < < max_total_weight < < " , reward " < < print_money ( best_reward )
2020-09-25 02:50:45 +02:00
< < " (including " < < print_money ( net_fee ) < < " in fees) " ) ;
2014-03-03 23:07:58 +01:00
return true ;
}
//---------------------------------------------------------------------------------
2016-01-29 16:09:17 +01:00
size_t tx_memory_pool : : validate ( uint8_t version )
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2018-07-18 23:24:53 +02:00
size_t tx_weight_limit = get_transaction_weight_limit ( version ) ;
2017-05-14 15:06:55 +02:00
std : : unordered_set < crypto : : hash > remove ;
2018-07-18 23:24:53 +02:00
m_txpool_weight = 0 ;
m_blockchain . for_all_txpool_txes ( [ this , & remove , tx_weight_limit ] ( const crypto : : hash & txid , const txpool_tx_meta_t & meta , const cryptonote : : blobdata * ) {
m_txpool_weight + = meta . weight ;
if ( meta . weight > tx_weight_limit ) {
LOG_PRINT_L1 ( " Transaction " < < txid < < " is too big ( " < < meta . weight < < " bytes), removing it from pool " ) ;
2017-05-14 15:06:55 +02:00
remove . insert ( txid ) ;
2017-03-22 21:21:54 +01:00
}
2017-03-22 21:26:56 +01:00
else if ( m_blockchain . have_tx ( txid ) ) {
LOG_PRINT_L1 ( " Transaction " < < txid < < " is in the blockchain, removing it from pool " ) ;
2017-05-14 15:06:55 +02:00
remove . insert ( txid ) ;
2017-03-22 21:21:54 +01:00
}
2017-05-14 15:06:55 +02:00
return true ;
2017-11-08 13:06:41 +01:00
} , false ) ;
2017-05-14 15:06:55 +02:00
size_t n_removed = 0 ;
if ( ! remove . empty ( ) )
{
LockedTXN lock ( m_blockchain ) ;
for ( const crypto : : hash & txid : remove )
{
try
2016-01-29 16:09:17 +01:00
{
2017-05-14 15:06:55 +02:00
cryptonote : : blobdata txblob = m_blockchain . get_txpool_tx_blob ( txid ) ;
cryptonote : : transaction tx ;
if ( ! parse_and_validate_tx_from_blob ( txblob , tx ) )
{
MERROR ( " Failed to parse tx from txpool " ) ;
continue ;
}
// remove tx from db first
m_blockchain . remove_txpool_tx ( txid ) ;
2018-07-18 23:24:53 +02:00
m_txpool_weight - = get_transaction_weight ( tx , txblob . size ( ) ) ;
2018-09-16 20:30:39 +02:00
remove_transaction_keyimages ( tx , txid ) ;
2017-05-14 15:06:55 +02:00
auto sorted_it = find_tx_in_sorted_container ( txid ) ;
if ( sorted_it = = m_txs_by_fee_and_receive_time . end ( ) )
{
LOG_PRINT_L1 ( " Removing tx " < < txid < < " from tx pool, but it was not found in the sorted txs container! " ) ;
}
else
{
m_txs_by_fee_and_receive_time . erase ( sorted_it ) ;
}
+ + n_removed ;
2016-01-29 16:09:17 +01:00
}
2017-05-14 15:06:55 +02:00
catch ( const std : : exception & e )
2016-01-29 16:09:17 +01:00
{
2017-05-14 15:06:55 +02:00
MERROR ( " Failed to remove invalid tx from pool " ) ;
// continue
2016-01-29 16:09:17 +01:00
}
}
2019-06-26 06:00:05 +02:00
lock . commit ( ) ;
2016-01-29 16:09:17 +01:00
}
2016-04-17 12:04:01 +02:00
if ( n_removed > 0 )
+ + m_cookie ;
2016-01-29 16:09:17 +01:00
return n_removed ;
}
//---------------------------------------------------------------------------------
2018-07-18 23:24:53 +02:00
bool tx_memory_pool : : init ( size_t max_txpool_weight )
2014-03-03 23:07:58 +01:00
{
2019-12-12 04:11:07 +01:00
auto locks = tools : : unique_locks ( m_transactions_lock , m_blockchain ) ;
2014-07-17 17:02:05 +02:00
2018-07-18 23:24:53 +02:00
m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT ;
2017-05-14 15:06:55 +02:00
m_txs_by_fee_and_receive_time . clear ( ) ;
m_spent_key_images . clear ( ) ;
2018-07-18 23:24:53 +02:00
m_txpool_weight = 0 ;
2017-09-20 19:55:16 +02:00
std : : vector < crypto : : hash > remove ;
2018-04-11 16:27:56 +02:00
// first add the not kept by block, then the kept by block,
// to avoid rejection due to key image collision
for ( int pass = 0 ; pass < 2 ; + + pass )
{
const bool kept = pass = = 1 ;
bool r = m_blockchain . for_all_txpool_txes ( [ this , & remove , kept ] ( const crypto : : hash & txid , const txpool_tx_meta_t & meta , const cryptonote : : blobdata * bd ) {
RPC overhaul
High-level details:
This redesigns the RPC layer to make it much easier to work with,
decouples it from an embedded HTTP server, and gets the vast majority of
the RPC serialization and dispatch code out of a very commonly included
header.
There is unfortunately rather a lot of interconnected code here that
cannot be easily separated out into separate commits. The full details
of what happens here are as follows:
Major details:
- All of the RPC code is now in a `cryptonote::rpc` namespace; this
renames quite a bit to be less verbose: e.g. CORE_RPC_STATUS_OK
becomes `rpc::STATUS_OK`, and `cryptonote::COMMAND_RPC_SOME_LONG_NAME`
becomes `rpc::SOME_LONG_NAME` (or just SOME_LONG_NAME for code already
working in the `rpc` namespace).
- `core_rpc_server` is now completely decoupled from providing any
request protocol: it is now *just* the core RPC call handler.
- The HTTP RPC interface now lives in a new rpc/http_server.h; this code
handles listening for HTTP requests and dispatching them to
core_rpc_server, then sending the results back to the caller.
- There is similarly a rpc/lmq_server.h for LMQ RPC code; more details
on this (and other LMQ specifics) below.
- RPC implementing code now returns the response object and throws when
things go wrong which simplifies much of the rpc error handling. They
can throw anything; generic exceptions get logged and a generic
"internal error" message gets returned to the caller, but there is
also an `rpc_error` class to return an error code and message used by
some json-rpc commands.
- RPC implementing functions now overload `core_rpc_server::invoke`
following the pattern:
RPC_BLAH_BLAH::response core_rpc_server::invoke(RPC_BLAH_BLAH::request&& req, rpc_context context);
This overloading makes the code vastly simpler: all instantiations are
now done with a small amount of generic instantiation code in a single
.cpp rather than needing to go to hell and back with a nest of epee
macros in a core header.
- each RPC endpoint is now defined by the RPC types themselves,
including its accessible names and permissions, in
core_rpc_server_commands_defs.h:
- every RPC structure now has a static `names()` function that returns
the names by which the end point is accessible. (The first one is
the primary, the others are for deprecated aliases).
- RPC command wrappers define their permissions and type by inheriting
from special tag classes:
- rpc::RPC_COMMAND is a basic, admin-only, JSON command, available
via JSON RPC. *All* JSON commands are now available via JSON RPC,
instead of the previous mix of some being at /foo and others at
/json_rpc. (Ones that were previously at /foo are still there for
backwards compatibility; see `rpc::LEGACY` below).
- rpc::PUBLIC specifies that the command should be available via a
restricted RPC connection.
- rpc::BINARY specifies that the command is not JSON, but rather is
accessible as /name and takes and returns values in the magic epee
binary "portable storage" (lol) data format.
- rpc::LEGACY specifies that the command should be available via the
non-json-rpc interface at `/name` for backwards compatibility (in
addition to the JSON-RPC interface).
- some epee serialization got unwrapped and de-templatized so that it
can be moved into a .cpp file with just declarations in the .h. (This
makes a *huge* difference for core_rpc_server_commands_defs.h and for
every compilation unit that includes it which previously had to
compile all the serialization code and then throw all by one copy away
at link time). This required some new macros so as to not break a ton
of places that will use the old way putting everything in the headers;
The RPC code uses this as does a few other places; there are comments
in contrib/epee/include/serialization/keyvalue_serialization.h as to
how to use it.
- Detemplatized a bunch of epee/storages code. Most of it should have
have been using templates at all (because it can only ever be called
with one type!), and now it isn't. This broke some things that didn't
properly compile because of missing headers or (in one case) a messed
up circular dependency.
- Significantly simplified a bunch of over-templatized serialization
code.
- All RPC serialization definitions is now out of
core_rpc_server_commands_defs.h and into a single .cpp file
(core_rpc_server_commands_defs.cpp).
- core RPC no longer uses the disgusting
BEGIN_URI_MAP2/MAP_URI_BLAH_BLAH macros. This was a terrible design
that forced slamming tons of code into a common header that didn't
need to be there.
- epee::struct_init is gone. It was a horrible hack that instiated
multiple templates just so the coder could be so lazy and write
`some_type var;` instead of properly value initializing with
`some_type var{};`.
- Removed a bunch of useless crap from epee. In particular, forcing
extra template instantiations all over the place in order to nest
return objects inside JSON RPC values is no longer needed, as are a
bunch of stuff related to the above de-macroization of the code.
- get_all_service_nodes, get_service_nodes, and get_n_service_nodes are
now combined into a single `get_service_nodes` (with deprecated
aliases for the others), which eliminates a fair amount of
duplication. The biggest obstacle here was getting the requested
fields reference passed through: this is now done by a new ability to
stash a context in the serialization object that can be retrieved by a
sub-serialized type.
LMQ-specifics:
- The LokiMQ instance moves into `cryptonote::core` rather than being
inside cryptonote_protocol. Currently the instance is used both for
qnet and rpc calls (and so needs to be in a common place), but I also
intend future PRs to use the batching code for job processing
(replacing the current threaded job queue).
- rpc/lmq_server.h handles the actual LMQ-request-to-core-RPC glue.
Unlike http_server it isn't technically running the whole LMQ stack
from here, but the parallel name with http_server seemed appropriate.
- All RPC endpoints are supported by LMQ under the same names as defined
generically, but prefixed with `rpc.` for public commands and `admin.`
for restricted ones.
- service node keys are now always available, even when not running in
`--service-node` mode: this is because we want the x25519 key for
being able to offer CURVE encryption for lmq RPC end-points, and
because it doesn't hurt to have them available all the time. In the
RPC layer this is now called "get_service_keys" (with
"get_service_node_key" as an alias) since they aren't strictly only
for service nodes. This also means code needs to check
m_service_node, and not m_service_node_keys, to tell if it is running
as a service node. (This is also easier to notice because
m_service_node_keys got renamed to `m_service_keys`).
- Added block and mempool monitoring LMQ RPC endpoints: `sub.block` and
`sub.mempool` subscribes the connection for new block and new mempool
TX notifications. The latter can notify on just blink txes, or all
new mempool txes (but only new ones -- txes dumped from a block don't
trigger it). The client gets pushed a [`notify.block`, `height`,
`hash`] or [`notify.tx`, `txhash`, `blob`] message when something
arrives.
Minor details:
- rpc::version_t is now a {major,minor} pair. Forcing everyone to pack
and unpack a uint32_t was gross.
- Changed some macros to constexprs (e.g. CORE_RPC_ERROR_CODE_...).
(This immediately revealed a couple of bugs in the RPC code that was
assigning CORE_RPC_ERROR_CODE_... to a string, and it worked because
the macro allows implicit conversion to a char).
- De-templatizing useless templates in epee (i.e. a bunch of templated
types that were never invoked with different types) revealed a painful
circular dependency between epee and non-epee code for tor_address and
i2p_address. This crap is now handled in a suitably named
`net/epee_network_address_hack.cpp` hack because it really isn't
trivial to extricate this mess.
- Removed `epee/include/serialization/serialize_base.h`. Amazingly the
code somehow still all works perfectly with this previously vital
header removed.
- Removed bitrotted, unused epee "crypted_storage" and
"gzipped_inmemstorage" code.
- Replaced a bunch of epee::misc_utils::auto_scope_leave_caller with
LOKI_DEFERs. The epee version involves quite a bit more instantiation
and is ugly as sin. Also made the `loki::defer` class invokable for
some edge cases that need calling before destruction in particular
conditions.
- Moved the systemd code around; it makes much more sense to do the
systemd started notification as in daemon.cpp as late as possible
rather than in core (when we can still have startup failures, e.g. if
the RPC layer can't start).
- Made the systemd short status string available in the get_info RPC
(and no longer require building with systemd).
- during startup, print (only) the x25519 when not in SN mode, and
continue to print all three when in SN mode.
- DRYed out some RPC implementation code (such as set_limit)
- Made wallet_rpc stop using a raw m_wallet pointer
2020-04-28 01:25:43 +02:00
if ( kept ! = ( bool ) meta . kept_by_block )
2018-04-11 16:27:56 +02:00
return true ;
2018-09-16 20:30:39 +02:00
cryptonote : : transaction_prefix tx ;
if ( ! parse_and_validate_tx_prefix_from_blob ( * bd , tx ) )
2018-04-11 16:27:56 +02:00
{
MWARNING ( " Failed to parse tx from txpool, removing " ) ;
remove . push_back ( txid ) ;
2018-09-25 11:57:24 +02:00
return true ;
2018-04-11 16:27:56 +02:00
}
2018-09-16 20:30:39 +02:00
if ( ! insert_key_images ( tx , txid , meta . kept_by_block ) )
2018-04-11 16:27:56 +02:00
{
MFATAL ( " Failed to insert key images from txpool tx " ) ;
return false ;
}
2019-01-25 04:15:52 +01:00
2019-11-29 04:36:43 +01:00
const bool non_standard_tx = ! tx . is_transfer ( ) ;
2019-01-25 04:15:52 +01:00
m_txs_by_fee_and_receive_time . emplace ( std : : tuple < bool , double , time_t > ( non_standard_tx , meta . fee / ( double ) meta . weight , meta . receive_time ) , txid ) ;
2018-07-18 23:24:53 +02:00
m_txpool_weight + = meta . weight ;
2018-04-11 16:27:56 +02:00
return true ;
} , true ) ;
if ( ! r )
2017-05-14 15:06:55 +02:00
return false ;
2018-04-11 16:27:56 +02:00
}
2017-09-20 19:55:16 +02:00
if ( ! remove . empty ( ) )
{
LockedTXN lock ( m_blockchain ) ;
for ( const auto & txid : remove )
{
try
{
m_blockchain . remove_txpool_tx ( txid ) ;
}
catch ( const std : : exception & e )
{
MWARNING ( " Failed to remove corrupt transaction: " < < txid ) ;
// ignore error
}
}
2019-06-26 06:00:05 +02:00
lock . commit ( ) ;
2017-09-20 19:55:16 +02:00
}
2016-04-17 12:04:01 +02:00
m_cookie = 0 ;
// Ignore deserialization error
2017-09-20 19:55:16 +02:00
return true ;
2014-03-03 23:07:58 +01:00
}
//---------------------------------------------------------------------------------
bool tx_memory_pool : : deinit ( )
{
2017-05-14 15:06:55 +02:00
return true ;
2014-03-03 23:07:58 +01:00
}
}