very basic tx construction, signing, and submission should be working

still def needs (much) testing

src/wallet3/daemon_comms.hpp

@@ -29,14 +29,14 @@ namespace wallet
      * if the wallet is already registered.
      *
      * A wallet should call this:
-     *   On creation, to inform the daemon comms that it exists and wishes to sync.
+     *   On creation, to inform the daemon comms that it exists and wishes to sync.  Pass new_wallet=true.
      *   If the wallet recieves blocks from daemon comms which are in the future for
      *   it.  In this case, it is telling the daemon comms to start syncing from
      *   earlier in the chain.  Pass check_sync_height=true.
      *   When the wallet finishes processing a batch of blocks.  Pass check_sync_height=false.
      */
     virtual void
-    register_wallet(Wallet& wallet, int64_t height, bool check_sync_height = false) = 0;
+    register_wallet(Wallet& wallet, int64_t height, bool check_sync_height = false, bool new_wallet = false) = 0;
 
     virtual void
     deregister_wallet(Wallet& wallet, std::promise<void>& p) = 0;

src/wallet3/db_schema.cpp

@@ -6,8 +6,14 @@
 #include <common/hex.h>
 #include <cryptonote_basic/cryptonote_basic.h>
 
+#include <iostream>
+
 namespace wallet
 {
+  WalletDB::~WalletDB()
+  {
+  }
+
   // FIXME: BLOB or TEXT for binary data below?
   void
   WalletDB::create_schema()
@@ -21,6 +27,21 @@ namespace wallet
     // TODO: table for balance "per account"
     db.exec(
         R"(
+          -- CHECK (id = 0) restricts this table to a single row
+          CREATE TABLE metadata (
+            id INTEGER NOT NULL PRIMARY KEY CHECK (id = 0),
+            db_version INTEGER NOT NULL DEFAULT 0,
+            balance INTEGER NOT NULL DEFAULT 0,
+            unlocked_balance INTEGER NOT NULL DEFAULT 0,
+            last_scan_height INTEGER NOT NULL DEFAULT -1,
+            scan_target_hash TEXT NOT NULL,
+            scan_target_height INTEGER NOT NULL DEFAULT 0,
+            output_count INTEGER NOT NULL DEFAULT 0
+          );
+
+          -- insert metadata row as default
+          INSERT INTO metadata VALUES (0,0,0,0,-1,"",0,0);
+
           CREATE TABLE blocks (
             height INTEGER NOT NULL PRIMARY KEY,
             transaction_count INTEGER NOT NULL,
@@ -28,6 +49,13 @@ namespace wallet
             timestamp INTEGER NOT NULL
           );
 
+          -- update scan height when new block added
+          CREATE TRIGGER block_added AFTER INSERT ON blocks
+          FOR EACH ROW
+          BEGIN
+            UPDATE metadata SET last_scan_height = NEW.height WHERE id = 0;
+          END;
+
           CREATE TABLE transactions (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
             block INTEGER NOT NULL REFERENCES blocks(height) ON DELETE CASCADE,
@@ -47,19 +75,6 @@ namespace wallet
           -- default "main" subaddress
           INSERT INTO subaddresses VALUES (0,0,TRUE);
 
-          -- CHECK (id = 0) restricts this table to a single row
-          CREATE TABLE metadata (
-            id INTEGER NOT NULL PRIMARY KEY CHECK (id = 0),
-            db_version INTEGER NOT NULL DEFAULT 0,
-            balance INTEGER NOT NULL DEFAULT 0,
-            unlocked_balance INTEGER NOT NULL DEFAULT 0,
-            last_scan_height INTEGER NOT NULL DEFAULT -1,
-            scan_target_hash TEXT NOT NULL,
-            scan_target_height INTEGER NOT NULL DEFAULT 0
-          );
-          -- insert metadata row as default
-          INSERT INTO metadata VALUES (0,0,0,0,-1,"",0);
-
           CREATE TABLE key_images (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
             key_image BLOB NOT NULL,
@@ -143,9 +158,24 @@ namespace wallet
     db_tx.commit();
   }
 
+  void
+  WalletDB::update_output_count(const int64_t count)
+  {
+    prepared_exec(
+        "UPDATE metadata SET output_count = output_count + ? WHERE id = 0;",
+        count);
+  }
+
   void
   WalletDB::store_block(const Block& block)
   {
+    int64_t output_count = 0;
+    for (const auto& tx : block.transactions)
+    {
+      output_count += tx.tx.vout.size();
+    }
+    update_output_count(output_count);
+
     prepared_exec(
         "INSERT INTO blocks(height,transaction_count,hash,timestamp) VALUES(?,?,?,?)",
         block.height,
@@ -297,8 +327,7 @@ namespace wallet
   int64_t
   WalletDB::chain_output_count()
   {
-    //TODO: this
-    return 0;
+    return prepared_get<int64_t>("SELECT output_count FROM metadata WHERE id=0;");
   }
 
 }  // namespace wallet

src/wallet3/db_schema.hpp

@@ -23,6 +23,8 @@ namespace wallet
   public:
     using db::Database::Database;
 
+    ~WalletDB();
+
     // Get a DB transaction.  This will revert any changes done to the db
     // while it exists when it is destroyed unless commit() is called on it.
     SQLite::Transaction db_transaction()
@@ -35,6 +37,10 @@ namespace wallet
     void
     create_schema();
 
+    // update the total output count, for decoy selection
+    void
+    update_output_count(const int64_t count);
+
     void
     store_block(const Block& block);
 

src/wallet3/default_daemon_comms.cpp

@@ -23,7 +23,6 @@ namespace wallet
       //TODO: error handling
       return;
     }
-    std::cout << "on_get_blocks_response() got " << response.size() - 1 << " blocks.\n";
 
     const auto& status = response[0];
     if (status != "OK" and status != "END")
@@ -117,7 +116,6 @@ namespace wallet
 
     int64_t start_height = blocks.front().height;
     int64_t end_height = blocks.back().height;
-    std::cout << "on_get_blocks_response() got blocks [" << start_height << " to " << end_height << "]\n";
 
     if (status == "END")
     {
@@ -417,12 +415,21 @@ namespace wallet
   }
 
   void
-  DefaultDaemonComms::register_wallet(wallet::Wallet& wallet, int64_t height, bool check_sync_height)
+  DefaultDaemonComms::register_wallet(wallet::Wallet& wallet, int64_t height, bool check_sync_height, bool new_wallet)
   {
-    omq->job([this,w=wallet.shared_from_this(),height,check_sync_height](){
-        wallets.insert_or_assign(w, height);
+    omq->job([this,w=wallet.shared_from_this(),height,check_sync_height,new_wallet](){
+        if (wallets.count(w))
+          wallets[w] = height;
+        else if (new_wallet)
+          wallets.emplace(w, height);
+
         if (check_sync_height)
-          sync_from_height = std::min(sync_from_height, height);
+        {
+          if (wallets.size() == 1) // if it's the only wallet
+            sync_from_height = height;
+          else
+            sync_from_height = std::min(sync_from_height, height);
+        }
         start_syncing();
         }, sync_thread);
   }
@@ -436,13 +443,27 @@ namespace wallet
   void
   DefaultDaemonComms::deregister_wallet(wallet::Wallet& wallet, std::promise<void>& p)
   {
-    omq->job([this,w=wallet.shared_from_this(),&p]() mutable {
+    auto dereg_finish = [this,&p]() mutable {
+      p.set_value();
+    };
+
+    omq->job([this, w=wallet.shared_from_this(), &p, dereg_finish]() mutable {
           wallets.erase(w);
           w.reset();
-          p.set_value();
+
+          // this fulfills the promise after any functions waiting on this thread
+          // have completed, so all references to wallet from here should be gone.
+          omq->job(dereg_finish, sync_thread);
           auto itr = std::min_element(wallets.begin(), wallets.end(),
               [](const auto& l, const auto& r){ return l.second < r.second; });
-          sync_from_height = itr->second;
+          if (itr != wallets.end())
+            sync_from_height = itr->second;
+          else
+          {
+            sync_from_height = 0;
+            syncing = false;
+          }
+
           std::cout << "deregister_wallet() setting sync_from_height to " << sync_from_height << "\n";
           if (sync_from_height != 0 and sync_from_height == top_block_height)
             syncing = false;

src/wallet3/default_daemon_comms.hpp

@@ -39,7 +39,7 @@ namespace wallet
     get_height() { return top_block_height; }
 
     void
-    register_wallet(wallet::Wallet& wallet, int64_t height, bool check_sync_height);
+    register_wallet(wallet::Wallet& wallet, int64_t height, bool check_sync_height, bool new_wallet);
 
     void
     deregister_wallet(Wallet& wallet, std::promise<void>& p);

src/wallet3/keyring.cpp

@@ -271,12 +271,20 @@ namespace wallet
       i++;
     }
 
+    /*
     // Sort the inputs by their key image
     std::sort(ptx.tx.vin.begin(), ptx.tx.vin.end(), [&](const auto& a, const auto& b) {
       const cryptonote::txin_to_key &tk0 = var::get<cryptonote::txin_to_key>(a);
       const cryptonote::txin_to_key &tk1 = var::get<cryptonote::txin_to_key>(b);
       return memcmp(&tk0.k_image, &tk1.k_image, sizeof(tk0.k_image)) > 0;
     });
+    */
+
+    // TODO: apply the above sorting
+
+    auto txkey_pub = secret_tx_key_to_public_tx_key(tx_key);
+    cryptonote::remove_field_from_tx_extra<cryptonote::tx_extra_pub_key>(ptx.tx.extra);
+    cryptonote::add_tx_extra<cryptonote::tx_extra_pub_key>(ptx.tx, txkey_pub);
 
     std::vector<rct::key> amount_keys;
     amount_keys.clear();
@@ -294,10 +302,13 @@ namespace wallet
       dest_keys.push_back(rct::pk2rct(out_eph_public_key));
       outamounts.push_back(recipient.amount);
       amount_out += recipient.amount;
+      ptx.tx.vout.push_back(out);
       // TODO sean the output should be shuffled
       i++;
     }
 
+    // TODO: extra pub keys as needed (will come into play with subaddresses)
+
     // Generate one time destination key for change address (Output Ephemeral Key)
     crypto::public_key change_out_eph_public_key = generate_change_address_ephemeral_keys(tx_key, ptx.change, i, amount_keys);
     cryptonote::tx_out change_out{};
@@ -307,8 +318,14 @@ namespace wallet
     change_out.target = change_tk;
     dest_keys.push_back(rct::pk2rct(change_out_eph_public_key));
     outamounts.push_back(ptx.change.amount);
+    ptx.tx.vout.push_back(change_out);
     amount_out += ptx.change.amount;
 
+    // Zero amounts in tx.vin and tx.vout before ringct step
+    for (auto& i : ptx.tx.vin)
+      var::get<cryptonote::txin_to_key>(i).amount = 0;
+    for (auto& o : ptx.tx.vout)
+      o.amount = 0;
 
     crypto::hash tx_prefix_hash = get_transaction_prefix_hash(ptx.tx);
 

src/wallet3/rpc/request_handler.cpp

@@ -3,9 +3,10 @@
 #include "commands.h"
 #include "command_parser.h"
 #include <wallet3/wallet.hpp>
-
 #include <wallet3/db_schema.hpp>
 
+#include <cryptonote_core/cryptonote_tx_utils.h>
+
 #include <unordered_map>
 #include <memory>
 
@@ -14,6 +15,7 @@ namespace wallet::rpc {
 
 using cryptonote::rpc::rpc_context;
 using cryptonote::rpc::rpc_request;
+using cryptonote::rpc::rpc_error;
 
   namespace {
 
@@ -83,14 +85,41 @@ void RequestHandler::invoke(SET_ACCOUNT_TAG_DESCRIPTION& command, rpc_context co
 }
 
 void RequestHandler::invoke(GET_HEIGHT& command, rpc_context context) {
-  auto height = wallet.db->scan_target_height();
-  command.response["height"] = height;
+  if (auto w = wallet.lock())
+  {
+    auto height = w->db->scan_target_height();
+    command.response["height"] = height;
 
-  //TODO: this
-  command.response["immutable_height"] = height;
+    //TODO: this
+    command.response["immutable_height"] = height;
+  }
 }
 
 void RequestHandler::invoke(TRANSFER& command, rpc_context context) {
+  if (auto w = wallet.lock())
+  {
+    // TODO: arg checking
+    // TODO: actually use all args
+
+    std::vector<cryptonote::tx_destination_entry> recipients;
+    for (const auto& [dest, amount] : command.request.destinations)
+    {
+      auto& entry = recipients.emplace_back();
+      cryptonote::address_parse_info addr_info;
+
+      if (not cryptonote::get_account_address_from_str(addr_info, w->nettype, dest))
+        //TODO: is 500 the "correct" error code?
+        throw rpc_error(500, std::string("Invalid destination: ") + dest);
+
+      entry.original = dest;
+      entry.amount = amount;
+      entry.addr = addr_info.address;
+      entry.is_subaddress = addr_info.is_subaddress;
+      entry.is_integrated = addr_info.has_payment_id;
+    }
+
+    auto ptx = w->tx_constructor->create_transaction(recipients);
+  }
 }
 
 void RequestHandler::invoke(TRANSFER_SPLIT& command, rpc_context context) {

src/wallet3/rpc/request_handler.h

@@ -10,6 +10,8 @@
 #include <string>
 #include <unordered_map>
 
+#include <memory>
+
 namespace wallet {
   class Wallet;
 }
@@ -36,11 +38,11 @@ extern const std::unordered_map<std::string, std::shared_ptr<const rpc_command>>
 
 class RequestHandler {
 
-  wallet::Wallet& wallet;
+  std::weak_ptr<wallet::Wallet> wallet;
 
 public:
 
-  RequestHandler(wallet::Wallet& wallet) : wallet(wallet) {};
+  RequestHandler(std::weak_ptr<wallet::Wallet> wallet) : wallet(wallet) {};
 
   void invoke(GET_BALANCE& command, rpc_context context);
   void invoke(GET_ADDRESS& command, rpc_context context);

src/wallet3/transaction_constructor.cpp

@@ -76,9 +76,9 @@ namespace wallet
     for (const auto& output : ptx.chosen_outputs)
     {
       indexes = (*decoy_selector)(output);
-      auto decoy_promise = daemon->fetch_decoys(indexes);
-      decoy_promise.wait();
-      ptx.decoys.emplace_back(decoy_promise.get());
+      auto decoy_future = daemon->fetch_decoys(indexes);
+      decoy_future.wait();
+      ptx.decoys.emplace_back(decoy_future.get());
     }
   }
 

src/wallet3/wallet.cpp

@@ -14,6 +14,8 @@
 
 #include <filesystem>
 #include <future>
+#include <chrono>
+#include <thread>
 
 #include <iostream>
 
@@ -32,7 +34,7 @@ namespace wallet
       , tx_scanner{keys, db}
       , tx_constructor{tx_constructor}
       , daemon_comms{daemon_comms}
-      , request_handler{*this}
+      , request_handler{weak_from_this()}
       , omq_server{request_handler}
   {
     if (not omq)
@@ -56,12 +58,13 @@ namespace wallet
   {
     omq->start();
     daemon_comms->set_remote("ipc://./oxend.sock");
-    daemon_comms->register_wallet(*this, last_scan_height + 1 /*next needed block*/, true);
+    daemon_comms->register_wallet(*this, last_scan_height + 1 /*next needed block*/,
+        true /* update sync height */,
+        true /* new wallet */);
   }
 
   Wallet::~Wallet()
   {
-    std::cout << "Wallet::~Wallet()\n";
   }
 
   uint64_t
@@ -133,13 +136,19 @@ namespace wallet
   void
   Wallet::deregister()
   {
-    auto self = weak_from_this();
     running = false;
+    auto self = weak_from_this();
     std::promise<void> p;
     auto f = p.get_future();
     daemon_comms->deregister_wallet(*this, p);
     f.wait();
+
+    /*
+    // At this point, only the true "owner" should have a reference
+    using namespace std::chrono_literals;
+    while (self.use_count() > 1)
+      std::this_thread::sleep_for(50ms);
+    */
   }
 
-
 }  // namespace wallet

src/wallet3/wallet.hpp

@@ -104,6 +104,8 @@ namespace wallet
     wallet::rpc::RequestHandler request_handler;
     wallet::rpc::OmqServer omq_server;
     bool running = true;
+
+    cryptonote::network_type nettype = cryptonote::TESTNET;
   };
 
 }  // namespace wallet

tests/wallet3_tests/jank_wallet.cpp

@@ -30,19 +30,39 @@ int main(void)
   auto comms = std::make_shared<wallet::DefaultDaemonComms>(oxenmq);
   cryptonote::address_parse_info senders_address{};
   cryptonote::get_account_address_from_str(senders_address, cryptonote::TESTNET, "T6Td9RNPPsMMApoxc59GLiVDS9a82FL2cNEwdMUCGWDLYTLv7e7rvi99aWdF4M2V1zN7q1Vdf1mage87SJ9gcgSu1wJZu3rFs");
-  auto ctor = std::make_shared<wallet::TransactionConstructor>(nullptr, comms, senders_address);
-  auto wallet = wallet::Wallet::create(oxenmq, keyring, ctor, comms, ":memory:", "");
+  auto wallet = wallet::Wallet::create(oxenmq, keyring, nullptr, comms, "test.sqlite", "");
 
   std::this_thread::sleep_for(2s);
   auto chain_height = comms->get_height();
 
   std::cout << "chain height: " << chain_height << "\n";
 
-  while (true)
+  int64_t old_height = -1;
+  int64_t scan_height = 0;
+
+  std::atomic<bool> done = false;
+
+  std::thread exit_thread([&](){
+      std::string foo;
+      std::cin >> foo;
+      done = true;
+      });
+
+  while (old_height != scan_height)
   {
     using namespace std::chrono_literals;
 
-    std::this_thread::sleep_for(1s);
-    std::cout << "after block " << wallet->last_scan_height << ", balance is: " << wallet->get_balance() << "\n";
+    old_height = scan_height;
+    scan_height = wallet->last_scan_height;
+    std::this_thread::sleep_for(5s);
+    std::cout << "after block " << scan_height << ", balance is: " << wallet->get_balance() << "\n";
+    if (done)
+      break;
   }
+
+  exit_thread.join();
+
+  std::cout << "scanning appears finished, scan height = " << wallet->last_scan_height << ", daemon comms height = " << comms->get_height() << "\n";
+
+  wallet->deregister();
 }

tests/wallet3_tests/tx_creation.cpp

@@ -3,6 +3,7 @@
 
 #include <wallet3/wallet.hpp>
 #include <wallet3/db_schema.hpp>
+#include <wallet3/transaction_scanner.hpp>
 
 #include <sqlitedb/database.hpp>
 
@@ -245,6 +246,23 @@ TEST_CASE("Transaction Signing", "[wallet,tx]")
       REQUIRE(decoys.size() == 10);
     }
 
+    wallet::TransactionScanner scanner{keys, wallet_with_valid_inputs.get_db()};
+
+    wallet::BlockTX btx;
+    btx.tx = signedtx;
+    btx.global_indices.resize(signedtx.vout.size());
+
+    auto recv = scanner.scan_received(btx, 123, 456);
+
+    REQUIRE(recv.size() == 1);
+    if (recv.size() == 1)
+      REQUIRE(recv[0].amount == 949969108610);
+
+    std::cout << __FILE__ << ":" << __LINE__ << " (" << __func__ << ") TODO sean remove this - scanner recv size: " << recv.size() << "\n";
+    std::cout << __FILE__ << ":" << __LINE__ << " (" << __func__ << ") TODO sean remove this - expected change amount: " << ptx.change.amount << "\n";
+    if (recv.size() == 1)
+      std::cout << __FILE__ << ":" << __LINE__ << " (" << __func__ << ") TODO sean remove this - scanner recv amount: " << recv[0].amount << "\n";
+
     std::cout << __FILE__ << ":" << __LINE__ << " (" << __func__ << ") TODO sean remove this - transaction hash: " << cryptonote::obj_to_json_str(ptx.tx.hash) << "\n";
     for (size_t n = 0; n < ptx.tx.vin.size(); ++n)
     {