oxen-io
/
oxen-core
mirror of https://github.com/oxen-io/oxen-core.git
2
1
Fork 0
Code Issues Projects Releases Wiki Activity

refactor wallet3 db usage 

db usage all now abstracted rather than having sql lying around everywhere.

fix a couple minor bugs in tx construction logic.
This commit is contained in:
Thomas Winget 2022-02-28 18:01:38 -05:00
parent 28f20c72c1
commit f67c0f8c4b
13 changed files with 304 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/sqlitedb/database.hpp
View File

@ -15,6 +15,7 @@
#include <thread>
#include <unordered_set>
#include <optional>
#include <string_view>
namespace db
{

164
src/wallet3/db_schema.cpp
View File

@ -1,10 +1,16 @@
#include "db_schema.hpp"
#include "output.hpp"
#include "block.hpp"
#include <common/hex.h>
#include <cryptonote_basic/cryptonote_basic.h>
namespace wallet
{
// FIXME: BLOB or TEXT for binary data below?
void
create_schema(SQLite::Database& db)
WalletDB::create_schema()
{
if (db.tableExists("outputs"))
return;
@ -136,4 +142,160 @@ namespace wallet
db_tx.commit();
}
void
WalletDB::store_block(const Block& block)
{
prepared_exec(
"INSERT INTO blocks(height,transaction_count,hash,timestamp) VALUES(?,?,?,?)",
block.height,
static_cast<int64_t>(block.transactions.size()),
tools::type_to_hex(block.hash),
block.timestamp);
}
void
WalletDB::store_transaction(
const crypto::hash& tx_hash, const int64_t height, const std::vector<Output>& outputs)
{
auto hash_str = tools::type_to_hex(tx_hash);
prepared_exec(
"INSERT INTO transactions(block,hash) VALUES(?,?)", height, hash_str);
for (const auto& output : outputs)
{
prepared_exec(
"INSERT INTO key_images(key_image) VALUES(?)", tools::type_to_hex(output.key_image));
prepared_exec(
R"(
INSERT INTO outputs(
amount,
output_index,
global_index,
unlock_time,
block_height,
tx,
output_key,
rct_mask,
key_image,
subaddress_major,
subaddress_minor)
VALUES(?,?,?,?,?,
(SELECT id FROM transactions WHERE hash = ?),
?,?,
(SELECT id FROM key_images WHERE key_image = ?),
?,?);
)",
output.amount,
output.output_index,
output.global_index,
output.unlock_time,
output.block_height,
hash_str,
tools::type_to_hex(output.key),
tools::type_to_hex(output.rct_mask),
tools::type_to_hex(output.key_image),
output.subaddress_index.major,
output.subaddress_index.minor);
}
}
void
WalletDB::store_spends(
const crypto::hash& tx_hash,
const int64_t height,
const std::vector<crypto::key_image>& spends)
{
auto hash_hex = tools::type_to_hex(tx_hash);
prepared_exec(
"INSERT INTO transactions(block,hash) VALUES(?,?) ON CONFLICT DO NOTHING",
height,
hash_hex);
for (const auto& key_image : spends)
{
prepared_exec(
R"(INSERT INTO spends(key_image, height, tx)
VALUES((SELECT id FROM key_images WHERE key_image = ?),
?,
(SELECT id FROM transactions WHERE hash = ?));)",
tools::type_to_hex(key_image),
height,
hash_hex);
}
}
int64_t
WalletDB::last_scan_height()
{
return prepared_get<int64_t>("SELECT last_scan_height FROM metadata WHERE id=0;");
}
int64_t
WalletDB::scan_target_height()
{
return prepared_get<int64_t>("SELECT scan_target_height FROM metadata WHERE id=0;");
}
void
WalletDB::update_top_block_info(int64_t height, const crypto::hash& hash)
{
prepared_exec("UPDATE metadata SET scan_target_height = ?, scan_target_hash = ? WHERE id = 0",
height, tools::type_to_hex(hash));
}
int64_t
WalletDB::overall_balance()
{
return prepared_get<int64_t>("SELECT balance FROM metadata WHERE id=0;");
}
int64_t
WalletDB::available_balance(std::optional<int64_t> min_amount)
{
std::string query = "SELECT sum(amount) FROM outputs WHERE spent_height = 0 AND spending = FALSE";
if (min_amount)
{
query += " AND amount > ?";
return prepared_get<int64_t>(query, *min_amount);
}
return prepared_get<int64_t>(query);
}
std::vector<Output>
WalletDB::available_outputs(std::optional<int64_t> min_amount)
{
std::vector<Output> outs;
std::string query = "SELECT amount, output_index, global_index, unlock_time, block_height, "
"spent_height, spending FROM outputs WHERE spent_height = 0 AND spending = FALSE ";
if (min_amount)
{
query += "AND amount > ? ";
}
query += "ORDER BY amount";
auto st = prepared_st(query);
if (min_amount)
st->bind(1, *min_amount);
while (st->executeStep())
{
outs.emplace_back(db::get<int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, int64_t>(st));
}
return outs;
}
int64_t
WalletDB::chain_output_count()
{
//TODO: this
return 0;
}
} // namespace wallet

77
src/wallet3/db_schema.hpp
View File

@ -1,9 +1,82 @@
#pragma once
#include <SQLiteCpp/SQLiteCpp.h>
#include <sqlitedb/database.hpp>
#include "output.hpp"
#include <optional>
namespace crypto
{
struct hash;
struct key_image;
}
namespace wallet
{
void
create_schema(SQLite::Database& db);
struct Output;
struct Block;
class WalletDB : public db::Database
{
public:
using db::Database::Database;
// Get a DB transaction. This will revert any changes done to the db
// while it exists when it is destroyed unless commit() is called on it.
SQLite::Transaction db_transaction()
{
return SQLite::Transaction{db};
}
// Create the database schema for the current version of the wallet db.
// Migration code will live elsewhere.
void
create_schema();
void
store_block(const Block& block);
void
store_transaction(const crypto::hash& tx_hash,
const int64_t height,
const std::vector<Output>& outputs);
void
store_spends(const crypto::hash& tx_hash,
const int64_t height,
const std::vector<crypto::key_image>& spends);
// The height of the last block added to the database.
int64_t
last_scan_height();
// The current chain height, as far as we know.
int64_t
scan_target_height();
// Update the top block height and hash.
void
update_top_block_info(int64_t height, const crypto::hash& hash);
// Get available balance across all subaddresses
int64_t
overall_balance();
// Get available balance with amount above an optional minimum amount.
// TODO: subaddress specification
int64_t
available_balance(std::optional<int64_t> min_amount);
// Selects all outputs with amount above an optional minimum amount.
// TODO: subaddress specification
std::vector<Output>
available_outputs(std::optional<int64_t> min_amount);
// Gets the total number of outputs on the chain. Since all Oxen outputs are RingCT
// and thus mixable, this can be used for decoy selection.
int64_t
chain_output_count();
};
}

2
src/wallet3/keyring.hpp
View File

@ -26,6 +26,8 @@ namespace wallet
, view_public_key(_view_public_key)
{}
Keyring() {}
virtual crypto::secret_key
generate_tx_key(uint8_t hf_version);

8
src/wallet3/rpc/omq_server.cpp
View File

@ -27,10 +27,10 @@ namespace wallet::rpc
using namespace cryptonote::rpc;
using oxenmq::AuthLevel;
OmqServer::OmqServer(std::shared_ptr<oxenmq::OxenMQ> omq, RequestHandler& request_handler)
: omq(omq)
, request_handler(request_handler)
void
OmqServer::set_omq(std::shared_ptr<oxenmq::OxenMQ> omq_in)
{
omq = omq_in;
//TODO: parametrize listening address(es) and auth
omq->listen_plain("ipc://./rpc.sock");
@ -41,7 +41,7 @@ OmqServer::OmqServer(std::shared_ptr<oxenmq::OxenMQ> omq, RequestHandler& reques
//omq->add_category("admin", oxenmq::AuthLevel::admin, 1 /* one reserved admin command thread */);
for (auto& cmd : rpc_commands) {
omq->add_request_command(cmd.second->is_restricted ? "restricted" : "rpc", cmd.first,
[name=std::string_view{cmd.first}, &call=*cmd.second, &request_handler, this](oxenmq::Message& m) {
[name=std::string_view{cmd.first}, &call=*cmd.second, this](oxenmq::Message& m) {
if (m.data.size() > 1)
m.send_reply(LMQ_BAD_REQUEST, "Bad request: RPC commands must have at most one data part "
"(received " + std::to_string(m.data.size()) + ")");

7
src/wallet3/rpc/omq_server.h
View File

@ -16,7 +16,12 @@ class OmqServer
public:
OmqServer(std::shared_ptr<oxenmq::OxenMQ> omq, RequestHandler& request_handler);
OmqServer(RequestHandler& request_handler) :
omq(nullptr), request_handler(request_handler)
{}
void
set_omq(std::shared_ptr<oxenmq::OxenMQ> omq);
};

4
src/wallet3/rpc/request_handler.cpp
View File

@ -4,7 +4,7 @@
#include "command_parser.h"
#include <wallet3/wallet.hpp>
#include <sqlitedb/database.hpp>
#include <wallet3/db_schema.hpp>
#include <unordered_map>
#include <memory>
@ -83,7 +83,7 @@ void RequestHandler::invoke(SET_ACCOUNT_TAG_DESCRIPTION& command, rpc_context co
}
void RequestHandler::invoke(GET_HEIGHT& command, rpc_context context) {
auto height = wallet.db->prepared_get<int64_t>("SELECT COUNT(*) FROM blocks;");
auto height = wallet.db->scan_target_height();
command.response["height"] = height;
//TODO: this

23
src/wallet3/transaction_constructor.cpp
View File

@ -3,7 +3,7 @@
#include "decoy.hpp"
#include "output_selection/output_selection.hpp"
#include "decoy_selection/decoy_selection.hpp"
#include <sqlitedb/database.hpp>
#include "db_schema.hpp"
namespace wallet
{
@ -46,8 +46,7 @@ namespace wallet
// as an additional (2nd+) input. Finally if the wallet balance is not sufficient
// allow the change to be dust but this will only occur if the wallet has enough to cover the
// transaction but not enough to also cover the dust which should be extremely unlikely.
int64_t wallet_balance = db->prepared_get<int>(
"SELECT sum(amount) FROM outputs WHERE amount > ?", additional_input * static_cast<int64_t>(ptx.fee_per_byte));
int64_t wallet_balance = db->available_balance(additional_input * static_cast<int64_t>(ptx.fee_per_byte));
if (wallet_balance < transaction_total)
throw std::runtime_error("Insufficient Wallet Balance");
else if (wallet_balance > transaction_total + single_input_size * static_cast<int64_t>(ptx.fee_per_byte))
@ -55,19 +54,8 @@ namespace wallet
else if (wallet_balance > transaction_total + additional_input * static_cast<int64_t>(ptx.fee_per_byte))
transaction_total += additional_input * ptx.fee_per_byte;
std::vector<Output> available_outputs{};
// Selects all outputs where the amount is greater than the estimated fee for an ADDITIONAL input.
SQLite::Statement st{
db->db,
"SELECT amount, output_index, global_index, unlock_time, block_height, spending, "
"spent_height FROM outputs WHERE amount > ? ORDER BY amount"};
st.bind(1, additional_input * static_cast<int64_t>(ptx.fee_per_byte));
while (st.executeStep())
{
wallet::Output o(db::get<int64_t, int64_t, int64_t, int64_t, int64_t, int64_t, int64_t>(st));
available_outputs.push_back(o);
}
auto available_outputs = db->available_outputs(additional_input * static_cast<int64_t>(ptx.fee_per_byte));
ptx.chosen_outputs = select_outputs(available_outputs, transaction_total);
ptx.fee = ptx.get_fee();
ptx.update_change();
@ -81,10 +69,7 @@ namespace wallet
{
ptx.decoys = {};
// This initialises the decoys to be selected from global_output_index= 0 to global_output_index = highest_output_index
// Oxen started with ringct transaction from its genesis so all transactions should be usable as decoys.
// We keep track of the number of transactions in each block so we can recreate the highest_output_index by summing
// all the transactions in every block.
int64_t max_output_index = db->prepared_get<int>("SELECT sum(transaction_count) FROM blocks;");
int64_t max_output_index = db->chain_output_count();
DecoySelector decoy_selection(0, max_output_index);
std::vector<int64_t> indexes;
for (const auto& output : ptx.chosen_outputs)

11
src/wallet3/transaction_constructor.hpp
View File

@ -7,17 +7,14 @@
#include "pending_transaction.hpp"
#include "daemon_comms.hpp"
namespace db
{
class Database;
}
namespace wallet
{
class WalletDB;
class TransactionConstructor
{
public:
TransactionConstructor(std::shared_ptr<db::Database> database, std::shared_ptr<DaemonComms> dmn)
TransactionConstructor(std::shared_ptr<WalletDB> database, std::shared_ptr<DaemonComms> dmn)
: db(std::move(database)), daemon(std::move(dmn))
{
std::tie(fee_per_byte, fee_per_output) = daemon->get_fee_parameters();
@ -42,7 +39,7 @@ namespace wallet
int64_t
estimate_fee() const;
std::shared_ptr<db::Database> db;
std::shared_ptr<WalletDB> db;
std::shared_ptr<DaemonComms> daemon;
};

125
src/wallet3/wallet.cpp
View File

@ -4,6 +4,7 @@
#include "wallet2½.hpp"
#include "block.hpp"
#include "block_tx.hpp"
#include "default_daemon_comms.hpp"
#include <common/hex.h>
#include <cryptonote_basic/cryptonote_basic.h>
@ -26,17 +27,28 @@ namespace wallet
std::string_view dbFilename,
std::string_view dbPassword)
: omq(omq)
, db{std::make_shared<db::Database>(std::filesystem::path(dbFilename), dbPassword)}
, db{std::make_shared<WalletDB>(std::filesystem::path(dbFilename), dbPassword)}
, keys{keys}
, tx_scanner{keys, db}
, tx_constructor{tx_constructor}
, daemon_comms{daemon_comms}
, request_handler{*this}
, omq_server{omq, request_handler}
, omq_server{request_handler}
{
create_schema(db->db);
last_scanned_height = db->prepared_get<int64_t>("SELECT last_scan_height FROM metadata WHERE id=0;");
scan_target_height = db->prepared_get<int64_t>("SELECT scan_target_height FROM metadata WHERE id=0;");
if (not omq)
{
omq = std::make_shared<oxenmq::OxenMQ>();
daemon_comms = std::make_shared<DefaultDaemonComms>(omq);
omq_server.set_omq(omq);
}
if (not daemon_comms)
daemon_comms = std::make_shared<DefaultDaemonComms>(omq);
if (not tx_constructor)
tx_constructor = std::make_shared<TransactionConstructor>(db, daemon_comms);
db->create_schema();
last_scan_height = db->last_scan_height();
scan_target_height = db->scan_target_height();
}
void
@ -44,7 +56,7 @@ namespace wallet
{
omq->start();
daemon_comms->set_remote("ipc://./oxend.sock");
daemon_comms->register_wallet(*this, last_scanned_height + 1 /*next needed block*/, true);
daemon_comms->register_wallet(*this, last_scan_height + 1 /*next needed block*/, true);
}
Wallet::~Wallet()
@ -55,37 +67,32 @@ namespace wallet
uint64_t
Wallet::get_balance()
{
return db->prepared_get<int64_t>("SELECT balance FROM metadata WHERE id=0;");
return db->overall_balance();
}
void
Wallet::add_block(const Block& block)
{
SQLite::Transaction db_tx(db->db);
auto db_tx = db->db_transaction();
db->prepared_exec(
"INSERT INTO blocks(height,transaction_count,hash,timestamp) VALUES(?,?,?,?)",
block.height,
static_cast<int64_t>(block.transactions.size()),
tools::type_to_hex(block.hash),
block.timestamp);
db->store_block(block);
for (const auto& tx : block.transactions)
{
if (auto outputs = tx_scanner.scan_received(tx, block.height, block.timestamp);
not outputs.empty())
{
store_transaction(tx.hash, block.height, outputs);
db->store_transaction(tx.hash, block.height, outputs);
}
if (auto spends = tx_scanner.scan_spent(tx.tx); not spends.empty())
{
store_spends(tx.hash, block.height, spends);
db->store_spends(tx.hash, block.height, spends);
}
}
db_tx.commit();
last_scanned_height++;
last_scan_height++;
}
void
@ -98,18 +105,18 @@ namespace wallet
//TODO: error handling; this shouldn't be able to happen
return;
if (blocks.front().height > last_scanned_height + 1)
if (blocks.front().height > last_scan_height + 1)
{
daemon_comms->register_wallet(*this, last_scanned_height + 1 /*next needed block*/, true);
daemon_comms->register_wallet(*this, last_scan_height + 1 /*next needed block*/, true);
return;
}
for (const auto& block : blocks)
{
if (block.height == last_scanned_height + 1)
if (block.height == last_scan_height + 1)
add_block(block);
}
daemon_comms->register_wallet(*this, last_scanned_height + 1 /*next needed block*/, false);
daemon_comms->register_wallet(*this, last_scan_height + 1 /*next needed block*/, false);
}
void
@ -118,9 +125,7 @@ namespace wallet
if (not running)
return;
auto hash_str = tools::type_to_hex(hash);
db->prepared_exec("UPDATE metadata SET scan_target_height = ?, scan_target_hash = ? WHERE id = 0",
height, hash_str);
db->update_top_block_info(height, hash);
scan_target_height = height;
}
@ -129,84 +134,12 @@ namespace wallet
Wallet::deregister()
{
auto self = weak_from_this();
std::cout << "Wallet ref count before deregister: " << self.use_count() << "\n";
running = false;
std::promise<void> p;
auto f = p.get_future();
daemon_comms->deregister_wallet(*this, p);
f.wait();
std::cout << "Wallet ref count after deregister: " << self.use_count() << "\n";
}
void
Wallet::store_transaction(
const crypto::hash& tx_hash, const int64_t height, const std::vector<Output>& outputs)
{
auto hash_str = tools::type_to_hex(tx_hash);
db->prepared_exec(
"INSERT INTO transactions(block,hash) VALUES(?,?)", height, hash_str);
for (const auto& output : outputs)
{
db->prepared_exec(
"INSERT INTO key_images(key_image) VALUES(?)", tools::type_to_hex(output.key_image));
db->prepared_exec(
R"(
INSERT INTO outputs(
amount,
output_index,
global_index,
unlock_time,
block_height,
tx,
output_key,
rct_mask,
key_image,
subaddress_major,
subaddress_minor)
VALUES(?,?,?,?,?,
(SELECT id FROM transactions WHERE hash = ?),
?,?,
(SELECT id FROM key_images WHERE key_image = ?),
?,?);
)",
output.amount,
output.output_index,
output.global_index,
output.unlock_time,
output.block_height,
hash_str,
tools::type_to_hex(output.key),
tools::type_to_hex(output.rct_mask),
tools::type_to_hex(output.key_image),
output.subaddress_index.major,
output.subaddress_index.minor);
}
}
void
Wallet::store_spends(
const crypto::hash& tx_hash,
const int64_t height,
const std::vector<crypto::key_image>& spends)
{
auto hash_hex = tools::type_to_hex(tx_hash);
db->prepared_exec(
"INSERT INTO transactions(block,hash) VALUES(?,?) ON CONFLICT DO NOTHING",
height,
hash_hex);
for (const auto& key_image : spends)
{
db->prepared_exec(
R"(INSERT INTO spends(key_image, height, tx)
VALUES((SELECT id FROM key_images WHERE key_image = ?),
?,
(SELECT id FROM transactions WHERE hash = ?));)",
tools::type_to_hex(key_image),
height,
hash_hex);
}
}
} // namespace wallet

20
src/wallet3/wallet.hpp
View File

@ -11,11 +11,6 @@
#include <memory>
#include <string_view>
namespace db
{
class Database;
}
namespace oxenmq
{
class OxenMQ;
@ -24,6 +19,8 @@ namespace oxenmq
namespace wallet
{
class WalletDB;
struct Block;
class Wallet : public std::enable_shared_from_this<Wallet>
@ -92,22 +89,13 @@ namespace wallet
deregister();
int64_t scan_target_height = 0;
int64_t last_scanned_height = -1;
int64_t last_scan_height = -1;
protected:
void
store_transaction(
const crypto::hash& tx_hash, const int64_t height, const std::vector<Output>& outputs);
void
store_spends(
const crypto::hash& tx_hash,
const int64_t height,
const std::vector<crypto::key_image>& spends);
std::shared_ptr<oxenmq::OxenMQ> omq;
std::shared_ptr<db::Database> db;
std::shared_ptr<WalletDB> db;
std::shared_ptr<Keyring> keys;
TransactionScanner tx_scanner;

10
tests/wallet3_tests/db_schema.cpp
View File

@ -5,15 +5,15 @@
TEST_CASE("DB Schema", "[wallet,db]")
{
db::Database db{std::filesystem::path(":memory:"), ""};
wallet::WalletDB db{std::filesystem::path(":memory:"), ""};
SECTION("db schema creation succeeds")
{
REQUIRE_NOTHROW(wallet::create_schema(db.db));
REQUIRE_NOTHROW(db.create_schema());
}
// will not throw if schema is already set up
REQUIRE_NOTHROW(wallet::create_schema(db.db));
REQUIRE_NOTHROW(db.create_schema());
REQUIRE(db.db.tableExists("blocks"));
@ -61,9 +61,9 @@ TEST_CASE("DB Schema", "[wallet,db]")
TEST_CASE("DB Triggers", "[wallet,db]")
{
db::Database db{std::filesystem::path(":memory:"), ""};
wallet::WalletDB db{std::filesystem::path(":memory:"), ""};
REQUIRE_NOTHROW(wallet::create_schema(db.db));
REQUIRE_NOTHROW(db.create_schema());
REQUIRE(db.db.tableExists("blocks"));
REQUIRE_NOTHROW(db.prepared_exec("INSERT INTO blocks VALUES(?,?,?,?);", 0, 0, "foo", 0));

9
tests/wallet3_tests/mock_wallet.hpp
View File

@ -3,6 +3,7 @@
#include <wallet3/wallet.hpp>
#include <wallet3/block.hpp>
#include <sqlitedb/database.hpp>
#include <wallet3/db_schema.hpp>
namespace wallet
{
@ -23,11 +24,11 @@ class MockWallet : public Wallet
{
public:
MockWallet() : Wallet({},{},{},{},":memory:",{}){};
MockWallet() : Wallet({},std::make_shared<Keyring>(),{},{},":memory:",{}){};
int64_t height = 0;
std::shared_ptr<db::Database> get_db() { return db; };
std::shared_ptr<WalletDB> get_db() { return db; };
void
store_test_transaction(const int64_t amount)
@ -48,8 +49,8 @@ class MockWallet : public Wallet
o.key_image = debug_random_filled<crypto::key_image>(height);
dummy_outputs.push_back(o);
SQLite::Transaction db_tx(db->db);
store_transaction(hash, height, dummy_outputs);
auto db_tx = db->db_transaction();
db->store_transaction(hash, height, dummy_outputs);
db_tx.commit();
};