Merge pull request #1632 from darcys22/parse-tx-extra-util

Adds util to parse tx extra messages

.drone.jsonnet

@@ -237,6 +237,43 @@ local gui_wallet_step_darwin = {
 
 
 [
+  // Static build to make wallet3:
+  {
+    name: 'Static (wallet3)',
+    kind: 'pipeline',
+    type: 'docker',
+    platform: { arch: 'amd64' },
+    steps: [{
+      name: 'build',
+      image: docker_base + 'ubuntu-lts',
+      pull: 'always',
+      environment: { SSH_KEY: { from_secret: 'SSH_KEY' } },
+      commands: submodules_commands + [
+        'apt update',
+        'eatmydata ' + apt_get_quiet + ' install -y --no-install-recommends cmake git ninja-build ccache '
+        + std.join(' ', static_build_deps),
+        'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y lsb-release',
+        'cp contrib/deb.oxen.io.gpg /etc/apt/trusted.gpg.d',
+        'echo deb http://deb.oxen.io $$(lsb_release -sc) main >/etc/apt/sources.list.d/oxen.list',
+        'eatmydata ' + apt_get_quiet + ' update',
+        'apt update',
+        'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y python3-venv python3-oxenmq',
+        'pip3 install --upgrade pip',
+        'pip3 install --upgrade build',
+        'pip3 install --upgrade setuptools',
+        'mkdir build',
+        'cd build',
+        'cmake .. -G Ninja ' +
+        '-DSTATIC=ON -DBUILD_STATIC_DEPS=ON -DUSE_LTO=OFF -DCMAKE_BUILD_TYPE=Release -DWARNINGS_AS_ERRORS=OFF',
+        'ninja -j6 -v wallet3_merged',
+        'pip3 install ./pybind/',
+        'cd ..',
+        'cd src/wallet3/cli-wallet/',
+        'python3.10 -m build',
+      ],
+    }],
+  },
+
   // Various debian builds
   debian_pipeline('Debian sid (w/ tests) (amd64)', docker_base + 'debian-sid', lto=true, run_tests=true),
   debian_pipeline('Debian sid Debug (amd64)', docker_base + 'debian-sid', build_type='Debug', cmake_extra='-DBUILD_DEBUG_UTILS=ON'),
@@ -353,4 +390,5 @@ local gui_wallet_step_darwin = {
       ],
     }],
   },
+
 ]

pybind/CMakeLists.txt

@@ -1,11 +1,13 @@
 pybind11_add_module(pywallet3 MODULE
   module.cpp
-  wallet/daemon_comms_config.cpp
-  wallet/rpc_config.cpp
   wallet/keyring.cpp
   wallet/keyring_manager.cpp
   wallet/wallet.cpp
   wallet/wallet_config.cpp
+  wallet/general_wallet_config.cpp
+  wallet/logging_config.cpp
+  wallet/daemon_comms_config.cpp
+  wallet/rpc_config.cpp
 )
 target_link_libraries(pywallet3 PUBLIC wallet3)
 target_include_directories(pywallet3 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})

pybind/common.hpp

@@ -16,6 +16,12 @@ namespace wallet
   void
   KeyringManager_Init(py::module& mod);
 
+  void
+  WalletConfig_Init(py::module& mod);
+
+  void
+  GeneralWalletConfig_Init(py::module& mod);
+
   void
   DaemonCommsConfig_Init(py::module& mod);
 
@@ -23,5 +29,6 @@ namespace wallet
   RPCConfig_Init(py::module& mod);
 
   void
-  WalletConfig_Init(py::module& mod);
+  LoggingConfig_Init(py::module& mod);
+
 }  // namespace wallet

pybind/module.cpp

@@ -5,7 +5,9 @@ PYBIND11_MODULE(pywallet3, m)
   wallet::Wallet_Init(m);
   wallet::Keyring_Init(m);
   wallet::KeyringManager_Init(m);
+  wallet::WalletConfig_Init(m);
+  wallet::GeneralWalletConfig_Init(m);
+  wallet::LoggingConfig_Init(m);
   wallet::DaemonCommsConfig_Init(m);
   wallet::RPCConfig_Init(m);
-  wallet::WalletConfig_Init(m);
 }

pybind/wallet/general_wallet_config.cpp

@@ -0,0 +1,15 @@
+#include "../common.hpp"
+#include "wallet3/config/config.hpp"
+
+namespace wallet
+{
+  void
+  GeneralWalletConfig_Init(py::module& mod)
+  {
+    py::class_<GeneralWalletConfig, std::shared_ptr<GeneralWalletConfig>>(mod, "GeneralWalletConfig")
+      .def(py::init<>())
+      .def_readwrite("datadir", &GeneralWalletConfig::datadir)
+      .def_readwrite("append_network_type_to_datadir", &GeneralWalletConfig::append_network_type_to_datadir);
+  }
+
+}  // namespace wallet

pybind/wallet/logging_config.cpp

@@ -0,0 +1,17 @@
+#include "../common.hpp"
+#include "wallet3/config/config.hpp"
+
+namespace wallet
+{
+  void
+  LoggingConfig_Init(py::module& mod)
+  {
+    py::class_<LoggingConfig, std::shared_ptr<LoggingConfig>>(mod, "LoggingConfig")
+      .def(py::init<>())
+      .def_readwrite("level", &LoggingConfig::level)
+      .def_readwrite("save_logs_in_subdirectory", &LoggingConfig::save_logs_in_subdirectory)
+      .def_readwrite("logdir", &LoggingConfig::logdir)
+      .def_readwrite("log_filename", &LoggingConfig::log_filename);
+  }
+
+}  // namespace wallet

pybind/wallet/wallet.cpp

@@ -5,6 +5,21 @@
 #include <wallet3/keyring.hpp>
 #include <wallet3/config/config.hpp>
 #include <oxenmq/oxenmq.h>
+#include <oxen/log.hpp>
+
+static auto logcat = oxen::log::Cat("omq");
+
+void omq_logger(oxenmq::LogLevel level, const char* file, int line, std::string message) {
+    constexpr std::string_view format = "[{}:{}]: {}";
+    switch (level) {
+      case oxenmq::LogLevel::fatal: oxen::log::critical(logcat, format, file, line, message); break;
+      case oxenmq::LogLevel::error: oxen::log::error(logcat, format, file, line, message); break;
+      case oxenmq::LogLevel::warn: oxen::log::warning(logcat, format, file, line, message); break;
+      case oxenmq::LogLevel::info: oxen::log::info(logcat, format, file, line, message); break;
+      case oxenmq::LogLevel::debug: oxen::log::debug(logcat, format, file, line, message); break;
+      case oxenmq::LogLevel::trace: oxen::log::trace(logcat, format, file, line, message); break;
+    }
+}
 
 namespace wallet
 {
@@ -15,12 +30,11 @@ namespace wallet
       .def(py::init([](const std::string& wallet_name, std::shared_ptr<Keyring> keyring, Config config) {
         auto& comms_config = config.daemon;
         auto& omq_rpc_config = config.omq_rpc;
-        auto oxenmq = std::make_shared<oxenmq::OxenMQ>();
-        auto comms = std::make_shared<DefaultDaemonComms>(std::move(oxenmq), comms_config);
-        return Wallet::create(oxenmq, std::move(keyring), nullptr, std::move(comms), wallet_name + ".sqlite", "", std::move(config));
+        auto oxenmq = std::make_shared<oxenmq::OxenMQ>(omq_logger, oxenmq::LogLevel::info);
+        auto comms = std::make_shared<DefaultDaemonComms>(oxenmq, comms_config);
+        return Wallet::create(std::move(oxenmq), std::move(keyring), nullptr, std::move(comms), wallet_name + ".sqlite", "", std::move(config));
       }))
       .def("get_balance", &Wallet::get_balance)
-      .def("get_unlocked_balance", &Wallet::get_unlocked_balance)
       .def("deregister", &Wallet::deregister);
   }
 

pybind/wallet/wallet_config.cpp

@@ -8,6 +8,8 @@ namespace wallet
   {
     py::class_<Config, std::shared_ptr<Config>>(mod, "WalletConfig")
       .def(py::init<>())
+      .def_readwrite("general", &Config::general)
+      .def_readwrite("logging", &Config::logging)
       .def_readwrite("daemon", &Config::daemon)
       .def_readwrite("omq_rpc", &Config::omq_rpc);
   }

src/checkpoints/checkpoints.cpp

@@ -96,7 +96,7 @@ namespace cryptonote
   {
     if (std::error_code ec; !fs::exists(json_hashfile_fullpath, ec))
     {
-      log::info(logcat, "Blockchain checkpoints file not found");
+      log::debug(logcat, "Blockchain checkpoints file not found");
       return true;
     }
 

src/common/string_util.h

@@ -154,7 +154,7 @@ T make_from_guts(std::string_view s) {
     if (s.size() != sizeof(T))
         throw std::runtime_error("Cannot reconstitute type: wrong type content size");
     T x;
-    std::memcpy(&x, s.data(), sizeof(T));
+    std::memcpy(static_cast<void*>(&x), s.data(), sizeof(T));
     return x;
 }
 

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -295,6 +295,7 @@ namespace cryptonote
   //---------------------------------------------------------------
   bool generate_key_image_helper_precomp(const account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const subaddress_index& received_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev)
   {
+    // This tries to compute the key image using a hardware device, if this succeeds return immediately, otherwise continue
     if (hwdev.compute_key_image(ack, out_key, recv_derivation, real_output_index, received_index, in_ephemeral, ki))
     {
       return true;

src/cryptonote_basic/hardfork.cpp

@@ -136,6 +136,13 @@ get_hard_fork_heights(network_type nettype, hf version) {
   return found;
 }
 
+hard_fork get_latest_hard_fork(network_type nettype) {
+  if (nettype == network_type::MAINNET) return mainnet_hard_forks.back();
+  if (nettype == network_type::TESTNET) return testnet_hard_forks.back();
+  if (nettype == network_type::FAKECHAIN) return fakechain_hardforks.back();
+  return devnet_hard_forks.back();
+}
+
 hf hard_fork_ceil(network_type nettype, hf version) {
   auto [it, end] = get_hard_forks(nettype);
   for (; it != end; it++)

src/cryptonote_basic/hardfork.h

@@ -55,6 +55,9 @@ namespace cryptonote
   std::pair<std::optional<uint64_t>, std::optional<uint64_t>>
   get_hard_fork_heights(network_type type, hf version);
 
+  // Returns the latest hardfork
+  hard_fork get_latest_hard_fork(network_type type);
+
   // Returns the lowest network version >= the given version, that is, it rounds up missing hf table
   // entries to the next largest entry.  Typically this returns the network version itself, but if
   // some versions are skipped (particularly on testnet/devnet/fakechain) then this will return the

src/cryptonote_core/oxen_name_system.cpp

@@ -523,6 +523,58 @@ bool bind_and_run(ons_sql_type type, sql_compiled_statement& statement, void *co
 
 
 } // end anonymous namespace
+  
+using namespace std::literals;
+
+using stringtypemap = std::pair<std::string_view, mapping_type>;
+static constexpr std::array ons_str_type_mappings = {
+    stringtypemap{"5"sv, mapping_type::lokinet_10years},
+    stringtypemap{"4"sv, mapping_type::lokinet_5years},
+    stringtypemap{"3"sv, mapping_type::lokinet_2years},
+    stringtypemap{"2"sv, mapping_type::lokinet},
+    stringtypemap{"1"sv, mapping_type::wallet},
+    stringtypemap{"0"sv, mapping_type::session},
+    stringtypemap{"session"sv, mapping_type::session},
+    stringtypemap{"wallet"sv,  mapping_type::wallet},
+    stringtypemap{"lokinet"sv, mapping_type::lokinet},
+    stringtypemap{"lokinet_2years"sv,  mapping_type::lokinet_2years},
+    stringtypemap{"lokinet_5years"sv,  mapping_type::lokinet_5years},
+    stringtypemap{"lokinet_10years"sv, mapping_type::lokinet_10years}
+};
+
+std::optional<mapping_type>
+parse_ons_type(std::string input)
+{
+  // Lower-case the input:
+  for (auto& c : input)
+      if (c >= 'A' && c <= 'Z')
+          c += ('A' - 'a');
+
+  for (const auto& [str, map] : ons_str_type_mappings)
+    if (str == input)
+      return map;
+
+  return std::nullopt;
+}
+
+using inttypemap = std::pair<uint16_t, mapping_type>;
+static constexpr std::array ons_int_type_mappings = {
+    inttypemap{5, mapping_type::lokinet_10years},
+    inttypemap{4, mapping_type::lokinet_5years},
+    inttypemap{3, mapping_type::lokinet_2years},
+    inttypemap{2, mapping_type::lokinet},
+    inttypemap{1, mapping_type::wallet},
+    inttypemap{0, mapping_type::session}
+};
+std::optional<mapping_type>
+parse_ons_type(uint16_t input)
+{
+  for (const auto& [inttype, map] : ons_int_type_mappings)
+    if (inttype == input)
+      return map;
+
+  return std::nullopt;
+}
 
 
 bool mapping_record::active(uint64_t blockchain_height) const

src/cryptonote_core/oxen_name_system.h

@@ -210,6 +210,12 @@ struct settings_record
   int          version;
 };
 
+std::optional<mapping_type>
+parse_ons_type(std::string input);
+
+std::optional<mapping_type>
+parse_ons_type(uint16_t input);
+
 struct mapping_record
 {
   // NOTE: We keep expired entries in the DB indefinitely because we need to

src/rpc/core_rpc_server.cpp

@@ -2894,60 +2894,63 @@ namespace cryptonote::rpc {
     test_trigger_uptime_proof.response["status"] = STATUS_OK;
   }
   //------------------------------------------------------------------------------------------------------------------------------
-  ONS_NAMES_TO_OWNERS::response core_rpc_server::invoke(ONS_NAMES_TO_OWNERS::request&& req, rpc_context context)
+  void core_rpc_server::invoke(ONS_NAMES_TO_OWNERS& ons_names_to_owners, rpc_context context)
   {
-    ONS_NAMES_TO_OWNERS::response res{};
 
     if (!context.admin)
-      check_quantity_limit(req.entries.size(), ONS_NAMES_TO_OWNERS::MAX_REQUEST_ENTRIES);
+    {
+      check_quantity_limit(ons_names_to_owners.request.name_hash.size(), ONS_NAMES_TO_OWNERS::MAX_REQUEST_ENTRIES);
+      check_quantity_limit(ons_names_to_owners.request.type.size(), ONS_NAMES_TO_OWNERS::MAX_TYPE_REQUEST_ENTRIES, "types");
+    }
 
     std::optional<uint64_t> height = m_core.get_current_blockchain_height();
     auto hf_version = get_network_version(nettype(), *height);
-    if (req.include_expired) height = std::nullopt;
 
     std::vector<ons::mapping_type> types;
+    types.clear();
+    if (types.capacity() < ons_names_to_owners.request.type.size())
+      types.reserve(ons_names_to_owners.request.type.size());
+    for (const auto type_str : ons_names_to_owners.request.type)
+    {
+      const auto maybe_type = ons::parse_ons_type(type_str);
+      if (!maybe_type.has_value())
+      {
+        ons_names_to_owners.response["status"] = "invalid type provided";
+        return;
+      }
+      types.push_back(*maybe_type);
+    }
+    ons_names_to_owners.response["type"] =ons_names_to_owners.request.type;
 
     ons::name_system_db &db = m_core.get_blockchain_storage().name_system_db();
-    for (size_t request_index = 0; request_index < req.entries.size(); request_index++)
+    for (size_t request_index = 0; request_index < ons_names_to_owners.request.name_hash.size(); request_index++)
     {
-      ONS_NAMES_TO_OWNERS::request_entry const &request = req.entries[request_index];
-      if (!context.admin)
-        check_quantity_limit(request.types.size(), ONS_NAMES_TO_OWNERS::MAX_TYPE_REQUEST_ENTRIES, "types");
-
-      types.clear();
-      if (types.capacity() < request.types.size())
-        types.reserve(request.types.size());
-      for (auto type : request.types)
-      {
-        types.push_back(static_cast<ons::mapping_type>(type));
-        if (!ons::mapping_type_allowed(hf_version, types.back()))
-          throw rpc_error{ERROR_WRONG_PARAM, "Invalid lokinet type '" + std::to_string(type) + "'"};
-      }
-
+      const auto& request = ons_names_to_owners.request.name_hash[request_index];
       // This also takes 32 raw bytes, but that is undocumented (because it is painful to pass
       // through json).
-      auto name_hash = ons::name_hash_input_to_base64(request.name_hash);
+      auto name_hash = ons::name_hash_input_to_base64(ons_names_to_owners.request.name_hash[request_index]);
       if (!name_hash)
         throw rpc_error{ERROR_WRONG_PARAM, "Invalid name_hash: expected hash as 64 hex digits or 43/44 base64 characters"};
 
-      std::vector<ons::mapping_record> records = db.get_mappings(types, *name_hash, height);
-      for (auto const &record : records)
+      std::vector<ons::mapping_record> record = db.get_mappings(types, *name_hash, height);
+      for (size_t type_index = 0; type_index < ons_names_to_owners.request.type.size(); type_index++)
       {
-        auto& entry = res.entries.emplace_back();
-        entry.entry_index                                      = request_index;
-        entry.type                                             = record.type;
-        entry.name_hash                                        = record.name_hash;
-        entry.owner                                            = record.owner.to_string(nettype());
-        if (record.backup_owner) entry.backup_owner            = record.backup_owner.to_string(nettype());
-        entry.encrypted_value                                  = oxenc::to_hex(record.encrypted_value.to_view());
-        entry.expiration_height                                = record.expiration_height;
-        entry.update_height                                    = record.update_height;
-        entry.txid                                             = tools::type_to_hex(record.txid);
+        auto& elem = ons_names_to_owners.response["result"].emplace_back();
+        elem["type"] = record[type_index].type;
+        elem["name_hash"] = record[type_index].name_hash;
+        elem["owner"] = record[type_index].owner.to_string(nettype());
+        if (record[type_index].backup_owner)
+          elem["backup_owner"] = record[type_index].backup_owner.to_string(nettype());
+        elem["encrypted_value"] = oxenc::to_hex(record[type_index].encrypted_value.to_view());
+        if (record[0].expiration_height)
+          elem["expiration_height"] = *(record[type_index].expiration_height);
+        elem["update_height"] = record[type_index].update_height;
+        elem["txid"] = tools::type_to_hex(record[type_index].txid);
       }
     }
 
-    res.status = STATUS_OK;
-    return res;
+
+    ons_names_to_owners.response["status"] = STATUS_OK;
   }
   //------------------------------------------------------------------------------------------------------------------------------
   void core_rpc_server::invoke(ONS_OWNERS_TO_NAMES& ons_owners_to_names, rpc_context context)

src/rpc/core_rpc_server.h

@@ -183,6 +183,7 @@ namespace cryptonote::rpc {
     void invoke(GET_OUTPUT_HISTOGRAM& get_output_histogram, rpc_context context);
     void invoke(ONS_OWNERS_TO_NAMES& ons_owners_to_names, rpc_context context);
     void invoke(GET_ACCRUED_BATCHED_EARNINGS& get_accrued_batched_earnings, rpc_context context);
+    void invoke(ONS_NAMES_TO_OWNERS& ons_names_to_owners, rpc_context context);
 
     // Deprecated Monero NIH binary endpoints:
     GET_ALT_BLOCKS_HASHES_BIN::response         invoke(GET_ALT_BLOCKS_HASHES_BIN::request&& req, rpc_context context);
@@ -198,7 +199,6 @@ namespace cryptonote::rpc {
 
     // FIXME: unconverted JSON RPC endpoints:
     GET_SERVICE_NODE_REGISTRATION_CMD::response         invoke(GET_SERVICE_NODE_REGISTRATION_CMD::request&& req, rpc_context context);
-    ONS_NAMES_TO_OWNERS::response                       invoke(ONS_NAMES_TO_OWNERS::request&& req, rpc_context context);
 
 private:
     bool check_core_ready();

src/rpc/core_rpc_server_command_parser.cpp

@@ -365,6 +365,12 @@ namespace cryptonote::rpc {
         "include_expired", ons_owners_to_names.request.include_expired);
   }
 
+  void parse_request(ONS_NAMES_TO_OWNERS& ons_names_to_owners, rpc_input in) {
+    get_values(in,
+        "name_hash", required{ons_names_to_owners.request.name_hash},
+        "type", ons_names_to_owners.request.type);
+  }
+
   void parse_request(GET_QUORUM_STATE& qs, rpc_input in) {
 
     get_values(in,

src/rpc/core_rpc_server_command_parser.h

@@ -38,6 +38,7 @@ namespace cryptonote::rpc {
   void parse_request(IS_KEY_IMAGE_SPENT& spent, rpc_input in);
   void parse_request(LOKINET_PING& lokinet_ping, rpc_input in);
   void parse_request(ONS_OWNERS_TO_NAMES& ons_owners_to_names, rpc_input in);
+  void parse_request(ONS_NAMES_TO_OWNERS& ons_names_to_owners, rpc_input in);
   void parse_request(ONS_RESOLVE& ons, rpc_input in);
   void parse_request(OUT_PEERS& out_peers, rpc_input in);
   void parse_request(POP_BLOCKS& pop_blocks, rpc_input in);

src/rpc/core_rpc_server_commands_defs.cpp

@@ -271,35 +271,4 @@ KV_SERIALIZE_MAP_CODE_BEGIN(GET_SERVICE_NODE_REGISTRATION_CMD::request)
   KV_SERIALIZE(staking_requirement)
 KV_SERIALIZE_MAP_CODE_END()
 
-
-KV_SERIALIZE_MAP_CODE_BEGIN(ONS_NAMES_TO_OWNERS::request_entry)
-  KV_SERIALIZE(name_hash)
-  KV_SERIALIZE(types)
-KV_SERIALIZE_MAP_CODE_END()
-
-
-KV_SERIALIZE_MAP_CODE_BEGIN(ONS_NAMES_TO_OWNERS::request)
-  KV_SERIALIZE(entries)
-  KV_SERIALIZE(include_expired)
-KV_SERIALIZE_MAP_CODE_END()
-
-
-KV_SERIALIZE_MAP_CODE_BEGIN(ONS_NAMES_TO_OWNERS::response_entry)
-  KV_SERIALIZE(entry_index)
-  KV_SERIALIZE_ENUM(type)
-  KV_SERIALIZE(name_hash)
-  KV_SERIALIZE(owner)
-  KV_SERIALIZE(backup_owner)
-  KV_SERIALIZE(encrypted_value)
-  KV_SERIALIZE(update_height)
-  KV_SERIALIZE(expiration_height)
-  KV_SERIALIZE(txid)
-KV_SERIALIZE_MAP_CODE_END()
-
-
-KV_SERIALIZE_MAP_CODE_BEGIN(ONS_NAMES_TO_OWNERS::response)
-  KV_SERIALIZE(entries)
-  KV_SERIALIZE(status)
-KV_SERIALIZE_MAP_CODE_END()
-
 }

src/rpc/core_rpc_server_commands_defs.h

@@ -2522,52 +2522,20 @@ namespace cryptonote::rpc {
   };
 
   OXEN_RPC_DOC_INTROSPECT
-  // Get the name mapping for a Loki Name Service entry. Loki currently supports mappings
-  // for Session and Lokinet.
+  // Get the name mapping for an Oxen Name Service entry. Oxen currently supports mappings
+  // for Session, Wallet and Lokinet.
   struct ONS_NAMES_TO_OWNERS : PUBLIC
   {
     static constexpr auto names() { return NAMES("ons_names_to_owners", "lns_names_to_owners"); }
 
     static constexpr size_t MAX_REQUEST_ENTRIES      = 256;
     static constexpr size_t MAX_TYPE_REQUEST_ENTRIES = 8;
-    struct request_entry
+
+    struct request_parameters
     {
-      std::string name_hash; // The 32-byte BLAKE2b hash of the name to resolve to a public key via Loki Name Service. The value must be provided either in hex (64 hex digits) or base64 (44 characters with padding, or 43 characters without).
-      std::vector<uint16_t> types; // If empty, query all types. Currently supported types are 0 (session) and 2 (lokinet). In future updates more mapping types will be available.
-
-      KV_MAP_SERIALIZABLE
-    };
-
-    struct request
-    {
-      std::vector<request_entry> entries; // Entries to look up
-      bool include_expired;               // Optional: if provided and true, include entries in the results even if they are expired
-
-      KV_MAP_SERIALIZABLE
-    };
-
-    struct response_entry
-    {
-      uint64_t entry_index;     // The index in request_entry's `entries` array that was resolved via Loki Name Service.
-      ons::mapping_type type;   // The type of Loki Name Service entry that the owner owns: currently supported values are 0 (session), 1 (wallet) and 2 (lokinet)
-      std::string name_hash;    // The hash of the name that was queried, in base64
-      std::string owner;        // The public key that purchased the Loki Name Service entry.
-      std::optional<std::string> backup_owner; // The backup public key that the owner specified when purchasing the Loki Name Service entry. Omitted if no backup owner.
-      std::string encrypted_value; // The encrypted value that the name maps to. See the `ONS_RESOLVE` description for information on how this value can be decrypted.
-      uint64_t update_height;   // The last height that this Loki Name Service entry was updated on the Blockchain.
-      std::optional<uint64_t> expiration_height; // For records that expire, this will be set to the expiration block height.
-      std::string txid;                          // The txid of the mapping's most recent update or purchase.
-
-      KV_MAP_SERIALIZABLE
-    };
-
-    struct response
-    {
-      std::vector<response_entry> entries;
-      std::string status; // Generic RPC error code. "OK" is the success value.
-
-      KV_MAP_SERIALIZABLE
-    };
+      std::vector<std::string> name_hash; // The 32-byte BLAKE2b hash of the name to resolve to a public key via Oxen Name Service. The value must be provided either in hex (64 hex digits) or base64 (44 characters with padding, or 43 characters without).
+      std::vector<uint16_t> type; // If empty, query all types. Currently supported types are 0 (session), 1 (wallet) and 2 (lokinet). In future updates more mapping types will be available.
+    } request;
   };
 
   /// RPC: ons/ons_owners_to_names
@@ -2848,14 +2816,14 @@ namespace cryptonote::rpc {
     SUBMIT_TRANSACTION,
     SYNC_INFO,
     TEST_TRIGGER_P2P_RESYNC,
-    TEST_TRIGGER_UPTIME_PROOF
+    TEST_TRIGGER_UPTIME_PROOF,
+    ONS_NAMES_TO_OWNERS
   >;
 
   using FIXME_old_rpc_types = tools::type_list<
     RELAY_TX,
     GET_OUTPUT_DISTRIBUTION,
-    GET_SERVICE_NODE_REGISTRATION_CMD,
-    ONS_NAMES_TO_OWNERS
+    GET_SERVICE_NODE_REGISTRATION_CMD
   >;
 
 } // namespace cryptonote::rpc

src/sqlitedb/database.hpp

@@ -16,6 +16,7 @@
 #include <string_view>
 
 #include "common/fs.h"
+#include <common/string_util.h>
 
 namespace db
 {
@@ -50,6 +51,36 @@ namespace db
     st.bindNoCopy(i, static_cast<const void*>(blob.data()), blob.size());
   }
 
+  // Wrapper for extracting BLOB values from a query without unnecessary copying.  This is intended
+  // to be called via `db::get` such as:
+  //
+  //    auto [num, data] = db::get<int, blob>(st);
+  //
+  // The `.data` string view here will point to the BLOB data directly.  Note that this view remains
+  // only value while the statement remains active, and so it must be used as needed immediately.
+  // This also means that this is *unsuitable* for one-shot methods like `prepared_get` (which
+  // finalize the statement before returning).
+  struct blob {
+      std::string_view data;
+      blob(SQLite::Column&& col)
+        : data{static_cast<const char*>(col.getBlob()), static_cast<size_t>(col.getBytes())} {}
+  };
+
+  // Takes a primitive struct from which we can directly initialize from the stored blob value.  The
+  // type `T` must be usable with `make_from_guts`.  Unlike `blob` this value *is* suitable for use
+  // in a one-shot method.
+  template <typename T>
+  struct blob_guts {
+      T value;
+      blob_guts(SQLite::Column&& col)
+          : value{tools::make_from_guts<T>(blob(std::move(col)).data)} {}
+
+      // Implicit rvalue-convertible to `T&&` so that you can use it somewhat transparently, for
+      // example, allowing implicit conversion from a `std::tuple<..., blob_guts<T>>` into a
+      // `std::tuple<..., T>`.
+      operator T&&() && { return std::move(value); }
+  };
+
   namespace detail
   {
     template <typename T>

src/wallet/wallet_rpc_server.cpp

@@ -3247,6 +3247,7 @@ namespace {
 
   ONS_KNOWN_NAMES::response wallet_rpc_server::invoke(ONS_KNOWN_NAMES::request&& req)
   {
+    //TODO sean this needs to fit the new request format
     require_open();
     ONS_KNOWN_NAMES::response res{};
 

src/wallet3/CMakeLists.txt

@@ -1,5 +1,5 @@
 add_library(wallet3
-  db_schema.cpp
+  db/walletdb.cpp
   default_daemon_comms.cpp
   keyring.cpp
   keyring_manager.cpp
@@ -26,6 +26,10 @@ target_link_libraries(wallet3
     cryptonote_core
     extra
     mnemonics
+    logging
+    oxen::logging
+    spdlog::spdlog
+    fmt::fmt
     SQLiteCpp)
 
 function(combine_archives output_archive)

src/wallet3/cli-wallet/Makefile

@@ -1,5 +1,5 @@
 PYTHON_MAJOR_VERSION=3
-PYTHON_MINOR_VERSION=8
+PYTHON_MINOR_VERSION=10
 PYTHON_VERSION=$(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION)
 PYTHON_WITH_VERSION=python$(PYTHON_VERSION)
 PIP_WITH_VERSION=pip$(PYTHON_VERSION)
@@ -7,12 +7,14 @@ PIP_WITH_VERSION=pip$(PYTHON_VERSION)
 all: build
 
 system_dependencies:
-	$(PIP_WITH_VERSION) install --upgrade setuptools
+	sudo apt install python3.10-venv python3-oxenmq
+	$(PIP_WITH_VERSION) install --upgrade pip
 	$(PIP_WITH_VERSION) install --upgrade build
+	$(PIP_WITH_VERSION) install --upgrade setuptools
 
 build:
 	$(PYTHON_WITH_VERSION) -m build
-	$(PIP_WITH_VERSION) install --editable .
+	$(PIP_WITH_VERSION) install --user --editable .
 
 run:
 	oxen_wallet_cli

src/wallet3/cli-wallet/README.md

@@ -1,13 +1,35 @@
 # Oxen Wallet CLI
 
+## Installing Dependancies
+```
+make system_dependencies
+```
+
+## Build using docker 
+```
+docker run --pull=always -v ~/oxen-core:/src --rm -it registry.oxen.rocks/lokinet-ci-debian-bullseye /bin/bash
+curl -so /etc/apt/trusted.gpg.d/oxen.gpg https://deb.oxen.io/pub.gpg
+echo "deb https://deb.oxen.io $(lsb_release -sc) main" | tee /etc/apt/sources.list.d/oxen.list
+apt update
+apt install gperf python3-venv python3-oxenmq
+pip3 install --upgrade pip
+pip3 install --upgrade build
+pip3 install --upgrade setuptools
+cd src/
+mkdir build
+cd build
+cmake -DBUILD_STATIC_DEPS=ON ..
+make wallet3_merged
+pip3 install ./pybind/
+cd ..
+cd src/wallet3/cli-wallet/
+python3.9 -m build
+pip3 install --user --editable .
+/usr/local/bin/oxen_wallet_cli
+```
+
 ## Development Notes
 ### Click stuff
 https://click.palletsprojects.com/en/8.1.x/
 
 https://openbase.com/python/click-repl
-
-### Example Python wallets
-
-https://github.com/AndreMiras/PyWallet
-
-https://github.com/Blockstream/green_cli

src/wallet3/cli-wallet/oxen_wallet_cli/context.py

@@ -1,27 +1,109 @@
 import atexit
 import sys
 
+from datetime import datetime, timedelta
+
+import json
+
 import pywallet3
+import oxenmq
 
 from oxen_wallet_cli import version
 
+
 class Context:
     """Holds global context related to the invocation of the tool"""
 
     def __init__(self):
         self.options = None
-        self.logged_in = False
         self.configured = False
         self.wallet = None
         self.wallet_core_config = None
         self.keyring_manager = None
+        self.omq = None
+        self.wallet_rpc = None
 
     def configure(self, options):
         self.options = options
         self.__dict__.update(options)
         self.wallet_core_config = pywallet3.WalletConfig()
         self.wallet_core_config.daemon.address = self.options["oxend_url"]
+        self.wallet_core_config.general.datadir = self.options["datadir"]
+        self.wallet_core_config.general.append_network_type_to_datadir = self.options["append_network_to_datadir"]
+        self.wallet_core_config.logging.level = self.options["log_level"]
         self.keyring_manager = pywallet3.KeyringManager(self.options["network"])
         self.configured = True
 
+    def omq_connection(self):
+        if self.omq is None:
+            self.omq = oxenmq.OxenMQ(log_level=oxenmq.LogLevel.warn)
+            self.omq.max_message_size = 200*1024*1024
+            self.omq.start()
+        if self.wallet_rpc is None:
+            # TODO sean make this dynamic from the datadir
+            omq_addr = oxenmq.Address("ipc:///home/sean/.oxen-wallet/testnet/{}".format(self.wallet_core_config.omq_rpc.sockname))
+            self.wallet_rpc = self.omq.connect_remote(omq_addr)
+
+    def rpc_future(self, endpoint, cache_seconds=3, *, cache_key='', args=None, fail_okay=True, timeout=10):
+        return FutureJSON(self.omq, self.wallet_rpc, endpoint, cache_seconds, cache_key=cache_key, args=args, fail_okay=fail_okay, timeout=timeout)
+
+cached = {}
+cached_args = {}
+cache_expiry = {}
+class FutureJSON():
+    """Class for making a OMQ JSON RPC request that uses a future to wait on the result, and caches
+    the results for a set amount of time so that if the same endpoint with the same arguments is
+    requested again the cache will be used instead of repeating the request.
+    Cached values are indexed by endpoint and optional key, and require matching arguments to the
+    previous call.  The cache_key should generally be a fixed value (*not* an argument-dependent
+    value) and can be used to provide multiple caches for different uses of the same endpoint.
+    Cache entries are *not* purged, they are only replaced, so using dynamic data in the key would
+    result in unbounded memory growth.
+    omq - the omq object
+    oxend - the oxend omq connection id object
+    endpoint - the omq endpoint, e.g. 'rpc.get_info'
+    cache_seconds - how long to cache the response; can be None to not cache it at all
+    cache_key - fixed string to enable different caches of the same endpoint
+    args - if not None, a value to pass (after converting to JSON) as the request parameter. Typically a dict.
+    fail_okay - can be specified as True to make failures silent (i.e. if failures are sometimes expected for this request)
+    timeout - maximum time to spend waiting for a reply
+    """
+
+    def __init__(self, omq, oxend, endpoint, cache_seconds=3, *, cache_key='', args=None, fail_okay=False, timeout=10):
+        self.endpoint = endpoint
+        self.cache_key = self.endpoint + cache_key
+        self.fail_okay = fail_okay
+        if args is not None:
+            args = json.dumps(args).encode()
+        if self.cache_key in cached and cached_args[self.cache_key] == args and cache_expiry[self.cache_key] >= datetime.now():
+            self.json = cached[self.cache_key]
+            self.args = None
+            self.future = None
+        else:
+            self.json = None
+            self.args = args
+            self.future = omq.request_future(oxend, self.endpoint, [] if self.args is None else [self.args], timeout=timeout)
+        self.cache_seconds = cache_seconds
+
+    def get(self):
+        """If the result is already available, returns it immediately (and can safely be called multiple times.
+        Otherwise waits for the result, parses as json, and caches it.  Returns None if the request fails"""
+        if self.json is None and self.future is not None:
+            try:
+                result = self.future.get()
+                self.future = None
+                if result[0] != b'200':
+                    raise RuntimeError("Request for {} failed: got {}".format(self.endpoint, result))
+                self.json = json.loads(result[1])
+                if self.cache_seconds is not None:
+                    cached[self.cache_key] = self.json
+                    cached_args[self.cache_key] = self.args
+                    cache_expiry[self.cache_key] = datetime.now() + timedelta(seconds=self.cache_seconds)
+            except RuntimeError as e:
+                if not self.fail_okay:
+                    print("Something getting wrong: {}".format(e), file=sys.stderr)
+                self.future = None
+
+        return self.json
+
 sys.modules[__name__] = Context()

src/wallet3/cli-wallet/oxen_wallet_cli/walletcli.py

@@ -1,23 +1,27 @@
-import logging
 import os
+from pathlib import Path
+import time
 
 import click
-from click_repl import repl
+import click_repl
+
+from tqdm import tqdm
 
 from oxen_wallet_cli import context
 
 import pywallet3
 
-def _get_config_dir(options):
-    """Return the default config dir for network"""
-    return os.path.expanduser(os.path.join('~', '.oxen-wallet', options['network']))
+OXEN_ATOMIC_UNITS = 1e9
 
 @click.group(invoke_without_command=True)
-@click.option('--log-level', type=click.Choice(['error', 'warn', 'info', 'debug']))
-@click.option('--network', default='testnet', help='Network: mainnet|testnet|devnet.')
-@click.option('--config-dir', '-C', default=None, help='Override config directory.')
+@click.option('--log-level', type=click.Choice(['error', 'warn', 'info', 'debug']), default="info")
+@click.option('--network', default='testnet', type=click.Choice(['mainnet', 'testnet', 'devnet'], case_sensitive=False), help='Network: mainnet|testnet|devnet.')
 @click.option('--oxend-url', default="ipc:///home/sean/.oxen/testnet/oxend.sock", type=str, help='Use the given daemon')
 @click.option('--datadir', help='A directory which the wallet will save data')
+@click.option('--rounding', help='how many decimal places will be displayed for oxen', type=int, default=2)
+@click.option('--append-network-to-datadir', default=True)
+@click.option('--wallet-name')
+# @click.option('--wallet-password')
 @click.pass_context
 def walletcli(click_ctx, **options):
     """Command line interface for Oxen Wallet CLI."""
@@ -25,29 +29,49 @@ def walletcli(click_ctx, **options):
         # In repl mode run configuration once only
         return
 
-    if options['log_level']:
-        py_log_level = {
-            'error': logging.ERROR,
-            'warn': logging.WARNING,
-            'info': logging.INFO,
-            'debug': logging.DEBUG,
-        }[options['log_level']]
-
-        logging.basicConfig(level=py_log_level)
-
-    if options['config_dir'] is None:
-        options['config_dir'] = _get_config_dir(options)
-    os.makedirs(options['config_dir'], exist_ok=True)
-
     if options['datadir'] is None:
-        options['datadir'] = os.path.join(options['config_dir'], 'oxen_datadir')
+        home = str(Path.home())
+        options['datadir'] = os.path.expanduser(os.path.join(home, '.oxen-wallet'))
+
     os.makedirs(options['datadir'], exist_ok=True)
+    if options['append_network_to_datadir']:
+        os.makedirs(os.path.expanduser(os.path.join(options['datadir'], options['network'])), exist_ok=True)
 
     context.configure(options)
 
     if click_ctx.invoked_subcommand is None:
-        click.echo("Run ':help' for help information, or ':quit' to quit.")
-        repl(click_ctx)
+        click.echo("Oxen wallet started, you will need to load a wallet to continue")
+        click.echo("Please use load-from-file or load-from-seed")
+        click.echo("Run 'help' for help information, or 'quit' to quit.")
+        click_repl.repl(click_ctx)
+
+def progress_bar():
+    click.echo("Starting Wallet Sync")
+    with tqdm(total=1, ncols = 80, nrows = 3, position = 0, leave=False, unit="blocks", colour="green") as pbar:
+        syncing = True
+        retries = 10
+        prev_height = 0
+        while syncing and retries > 0:
+            try:
+                status_future = context.rpc_future("rpc.status");
+                status_response = status_future.get();
+                pbar.total = status_response["target_height"]
+                pbar.update(status_response["sync_height"] - prev_height)
+                prev_height = status_response["sync_height"]
+                syncing = status_response["syncing"]
+                time.sleep(0.5)
+            except Excepiton as e:
+                retries -= 1
+    click.echo("Wallet Synced")
+    pbar.close()
+
+def display_status():
+    status_future = context.rpc_future("rpc.status");
+    status_response = status_future.get();
+    if status_response["syncing"]:
+        progress_bar()
+    else:
+        click.echo("Wallet Synced")
 
 @walletcli.command()
 def load_test_wallet():
@@ -62,10 +86,15 @@ def load_test_wallet():
     view_pub = "8a0ebacd613e0b03b8f27bc64bd961ea2ebf4c671c6e7f3268651acf0823fed5"
 
     keyring = pywallet3.Keyring(spend_priv, spend_pub, view_priv, view_pub, context.options["network"])
-    click.echo("Wallet address {} loaded".format(keyring.get_main_address()))
-    name = click.prompt("Wallet Name", default="{}-oxen-wallet".format(context.options["network"])).strip()
+    click.echo("Wallet address " + click.style("{}", fg='cyan', bold=True).format(keyring.get_main_address()) + " loaded")
+    if context.options['wallet_name'] is None:
+        name = click.prompt("Wallet Name", default="{}-oxen-wallet".format(context.options["network"])).strip()
+    else:
+        name = context.options['wallet_name']
     context.wallet_core_config.omq_rpc.sockname = name + ".sock";
     context.wallet = pywallet3.Wallet(name, keyring, context.wallet_core_config)
+    context.omq_connection()
+    display_status()
 
 @walletcli.command()
 @click.argument('seed_phrase', nargs=25)
@@ -79,9 +108,31 @@ def load_from_seed(seed_phrase, seed_phrase_passphrase):
     seed_phrase_str = ' '.join(seed_phrase)
     keyring = context.keyring_manager.generate_keyring_from_electrum_seed(seed_phrase_str, seed_phrase_passphrase)
     click.echo("Wallet address {} loaded".format(keyring.get_main_address()))
-    name = click.prompt("Wallet Name", default="{}-oxen-wallet".format(context.options["network"])).strip()
+    if context.options['wallet_name'] is None:
+        name = click.prompt("Wallet Name", default="{}-oxen-wallet".format(context.options["network"])).strip()
+    else:
+        name = context.options['wallet_name']
     context.wallet_core_config.omq_rpc.sockname = name + ".sock";
     context.wallet = pywallet3.Wallet(name, keyring, context.wallet_core_config)
+    context.omq_connection()
+    display_status()
+
+@walletcli.command()
+def load_from_file():
+    click.echo("Loading wallet from file")
+    if context.wallet is not None:
+        click.echo("Wallet already loaded")
+        return
+
+    keyring = None
+    if context.options['wallet_name'] is None:
+        name = click.prompt("Wallet Name", default="{}-oxen-wallet".format(context.options["network"])).strip()
+    else:
+        name = context.options['wallet_name']
+    context.wallet_core_config.omq_rpc.sockname = name + ".sock";
+    context.wallet = pywallet3.Wallet(name, keyring, context.wallet_core_config)
+    context.omq_connection()
+    display_status()
 
 @walletcli.command()
 def register_service_node():
@@ -92,18 +143,125 @@ def register_service_node():
         click.echo("The wallet address to be used is: {}".format(name))
         click.echo("TODO: This function is not yet implemented")
 
+@walletcli.command()
+def status():
+    if context.wallet is None:
+        click.echo("Wallet not loaded")
+        return
+    status_future = context.rpc_future("rpc.status");
+    status_response = status_future.get();
+    click.echo("Status: {}".format(status_response))
+
 @walletcli.command()
 def address():
-    # click.echo("Address: {}".format(context.keyring.get_main_address()))
-    click.echo("Address: {}".format("TODO sean get the address here"))
+    if context.wallet is None:
+        click.echo("Wallet not loaded")
+        return
+    get_address_future = context.rpc_future("rpc.get_address");
+    get_address_response = get_address_future.get();
+    address = get_address_response['address']
+    click.echo("Address: {}".format(address))
 
 @walletcli.command()
-def get_balance():
-    click.echo("Balance: {}".format(context.wallet.get_balance()))
+def balance():
+    if context.wallet is None:
+        click.echo("Wallet not loaded")
+        return
+    get_balance_future = context.rpc_future("rpc.get_balance");
+    get_balance_response = get_balance_future.get();
+    balance = get_balance_response['balance']
+    click.echo("Balance: {:.{oxen_precision}f} Oxen".format(balance/OXEN_ATOMIC_UNITS, oxen_precision=context.options["rounding"]))
 
 @walletcli.command()
-def get_unlocked_balance():
-    click.echo("Unlocked Balance: {}".format(context.wallet.get_unlocked_balance()))
+def unlocked_balance():
+    if context.wallet is None:
+        click.echo("Wallet not loaded")
+        return
+    get_balance_future = context.rpc_future("rpc.get_balance");
+    get_balance_response = get_balance_future.get();
+    unlocked_balance = get_balance_response['unlocked_balance']
+    click.echo("Unlocked Balance: {:.{oxen_precision}f} Oxen".format(unlocked_balance/OXEN_ATOMIC_UNITS, oxen_precision=context.options["rounding"]))
+
+@walletcli.command()
+def height():
+    height_future = context.rpc_future("rpc.get_height");
+    height = height_future.get();
+    click.echo("Height: {}".format(height))
+
+@walletcli.command()
+def transfer():
+    address = click.prompt("Enter the destination wallet address", default="").strip()
+    amount = click.prompt("Enter the amount in oxen to be sent to {}".format(address), default=0.0)
+    if address == "" or amount == 0.0:
+        click.prompt("Invalid address/amount entered")
+        return
+    amount_in_atomic_units = round(amount * OXEN_ATOMIC_UNITS, 0);
+    destination = {"address": address, "amount": amount_in_atomic_units}
+    transfer_params = {"destinations": [destination]}
+    transfer_future = context.rpc_future("restricted.transfer", args=transfer_params);
+    transfer_response = transfer_future.get();
+    click.echo("Transfer Response: {}".format(transfer_response))
+
+lokinet_years_dict = {"1": "lokinet", "2": "lokinet_2years", "5": "lokinet_5years", "10": "lokinet_10years"}
+
+# TODO better names for these ONS commands
+@walletcli.command()
+def ons_buy_mapping():
+    ons_type = click.prompt("What type of mapping would you like", type=click.Choice(['session', 'wallet', 'lokinet']), default="session").strip()
+    if ons_type == "lokinet":
+        lokinet_years = click.prompt("How many years would you like the lokinet mapping for?", type=click.Choice(["1", "2", "5", "10"]), default="1").strip()
+        ons_type = lokinet_years_dict[lokinet_years]
+
+    ons_name = click.prompt("Please enter the ons name you would like to register", default="").strip()
+    ons_value = click.prompt("Please enter the value for the ons mapping", default="").strip()
+    ons_buy_params = {
+            "name": ons_name,
+            "value": ons_value,
+            "type": ons_type,
+           }
+    ons_owner = click.prompt("Optional: Enter the address of a different owner", default="").strip()
+    if len(ons_owner) > 0:
+        ons_buy_params["owner"] = ons_owner
+    ons_backup_owner = click.prompt("Optional: Enter the address of a backup owner", default="").strip()
+    if len(ons_backup_owner) > 0:
+        ons_buy_params["backup_owner"] = ons_backup_owner
+
+    transfer_future = context.rpc_future("restricted.ons_buy_mapping", args=ons_buy_params);
+    transfer_response = transfer_future.get();
+    click.echo("ONS Buy Mapping Response: {}".format(transfer_response))
+
+# TODO better names for these ONS commands
+@walletcli.command()
+def ons_update_mapping():
+    ons_name = click.prompt("Please enter the ons name you would like to update", default="").strip()
+    ons_type = click.prompt("Please enter the type of ONS mapping this is", type=click.Choice(['session', 'wallet', 'lokinet', 'lokinet_2years', 'lokinet_5years', 'lokinet_10years']), default="session").strip()
+    ons_update_params = {
+            "name": ons_name,
+            "type": ons_type,
+           }
+    ons_value = click.prompt("Optional: Please enter a value to modify the ons mapping", default="").strip()
+    if len(ons_value) > 0:
+        ons_buy_params["value"] = ons_value
+    ons_owner = click.prompt("Optional: Please enter an address to modify the owner", default="").strip()
+    if len(ons_owner) > 0:
+        ons_buy_params["owner"] = ons_owner
+    ons_backup_owner = click.prompt("Optional: Please enter an address to modify the backup owner", default="").strip()
+    if len(ons_backup_owner) > 0:
+        ons_buy_params["backup_owner"] = ons_backup_owner
+
+    transfer_future = context.rpc_future("restricted.ons_update_mapping", args=ons_update_params);
+    transfer_response = transfer_future.get();
+    click.echo("ONS Update Mapping Response: {}".format(transfer_response))
+
+@walletcli.command()
+def quit():
+    if context.wallet:
+        context.wallet.deregister()
+    click_repl.exit()
+
+@walletcli.command()
+def help():
+    click.echo("TODO help")
 
 def main():
     walletcli()

src/wallet3/cli-wallet/pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "Click",
     "click-repl",
     "pywallet3",
+    "tqdm",
 ]
 dynamic = ["version"]
 

src/wallet3/config/config.hpp

@@ -3,6 +3,26 @@
 namespace wallet
 {
 
+  struct GeneralWalletConfig
+  {
+    std::string nettype = "testnet";                     // What network the wallet is operating on ("mainnet" | "testnet" | "devnet")
+    std::string datadir = "oxen-wallet";                 // Directory to store data (Database files, websocket file, logs)
+    bool append_network_type_to_datadir = true;          // If you specify a datadir do you want the wallet to save into subdirs for testnet 
+    uint32_t subaddress_lookahead_major = 50;            // The wallet will generate a number of accounts based on this figure
+    uint32_t subaddress_lookahead_minor = 200;           // The wallet will generate a number of addresses for each account based on this figure
+  };
+
+  struct LoggingConfig
+  {
+    std::string level = "info";
+    bool save_logs_in_subdirectory = true;            // e.g ~/.oxen-wallet/testnet/logs/wallet_logs.txt vs ~/.oxen-wallet/testnet/wallet_logs.txt
+    std::string logdir = "logs";                      // Directory to store log data
+    std::string log_filename = "wallet_logs.txt";     // name for logs
+    size_t log_file_size_limit = 1024 * 1024 * 50;    // 50MiB
+    size_t extra_files = 1;
+    bool rotate_on_open = true;                       // wallet will create a new log file every time its opened
+  };
+
   struct DaemonCommsConfig
   {
     std::string address;              // The remote url of the daemon.
@@ -24,6 +44,8 @@ namespace wallet
 
   struct Config
   {
+    GeneralWalletConfig general;
+    LoggingConfig logging;
     DaemonCommsConfig daemon;
     wallet::rpc::Config omq_rpc;
   };

src/wallet3/daemon_comms.hpp

@@ -56,6 +56,9 @@ namespace wallet
 
     virtual std::future<std::string>
     submit_transaction(const cryptonote::transaction& tx, bool blink) = 0;
+
+    virtual std::future<std::pair<std::string, crypto::hash>>
+    ons_names_to_owners(const std::string& name_hash, uint16_t type) = 0;
   };
 
 }  // namespace wallet

src/wallet3/db/walletdb.cpp

@@ -1,9 +1,9 @@
-#include "db_schema.hpp"
+#include "walletdb.hpp"
 
-#include "output.hpp"
-#include "block.hpp"
+#include "wallet3/block.hpp"
 
 #include <common/hex.h>
+#include <common/string_util.h>
 #include <cryptonote_basic/cryptonote_basic.h>
 
 #include <fmt/core.h>
@@ -11,6 +11,8 @@
 
 namespace wallet
 {
+  static auto logcat = oxen::log::Cat("wallet");
+
   WalletDB::~WalletDB()
   {
   }
@@ -37,21 +39,27 @@ namespace wallet
     // TODO: table for balance "per account"
     db.exec(
         R"(
-          -- CHECK (id = 0) restricts this table to a single row
           CREATE TABLE metadata (
-            id INTEGER NOT NULL PRIMARY KEY CHECK (id = 0),
-            db_version INTEGER NOT NULL DEFAULT 0,
-            nettype TEXT NOT NULL DEFAULT "testnet",
-            balance INTEGER NOT NULL DEFAULT 0,
-            unlocked_balance INTEGER NOT NULL DEFAULT 0,
-            last_scan_height INTEGER NOT NULL DEFAULT -1,
-            scan_target_hash TEXT NOT NULL,
-            scan_target_height INTEGER NOT NULL DEFAULT 0,
-            output_count INTEGER NOT NULL DEFAULT 0
-          );
+            id TEXT PRIMARY KEY NOT NULL,
+            val_numeric INT,
+            val_binary BLOB,
+            val_text TEXT,
+            -- Exactly one val_* must be set:
+            CHECK((val_numeric IS NOT NULL) + (val_binary IS NOT NULL) + (val_text IS NOT NULL) == 1)
+          ) STRICT;
 
-          -- insert metadata row as default
-          INSERT INTO metadata VALUES (0,0,"testnet",0,0,-1,"",0,0);
+          INSERT INTO metadata(id, val_numeric)
+          VALUES
+            ('db_version', 0),
+            ('balance', 0),
+            ('last_scan_height', 0),
+            ('scan_target_height', 0),
+            ('output_count', 0);
+
+          INSERT INTO metadata(id, val_text)
+          VALUES
+            ('nettype', 'testnet'),
+            ('scan_target_hash', '');
 
           CREATE TABLE blocks (
             height INTEGER NOT NULL PRIMARY KEY,
@@ -64,16 +72,16 @@ namespace wallet
           CREATE TRIGGER block_added AFTER INSERT ON blocks
           FOR EACH ROW
           BEGIN
-            UPDATE metadata SET last_scan_height = NEW.height WHERE id = 0;
-            UPDATE metadata SET output_count = output_count + NEW.output_count WHERE id = 0;
+            UPDATE metadata SET val_numeric = NEW.height WHERE id = 'last_scan_height';
+            UPDATE metadata SET val_numeric = val_numeric + NEW.output_count WHERE id = 'output_count';
           END;
 
           -- update scan height when new block removed
           CREATE TRIGGER block_removed AFTER DELETE ON blocks
           FOR EACH ROW
           BEGIN
-            UPDATE metadata SET last_scan_height = OLD.height - 1 WHERE id = 0;
-            UPDATE metadata SET output_count = output_count - OLD.output_count WHERE id = 0;
+            UPDATE metadata SET val_numeric = OLD.height - 1 WHERE id = 'last_scan_height';
+            UPDATE metadata SET val_numeric = val_numeric - OLD.output_count WHERE id = 'last_scan_height';
           END;
 
           CREATE TABLE transactions (
@@ -117,8 +125,7 @@ namespace wallet
             rct_mask BLOB NOT NULL,
             key_image INTEGER NOT NULL REFERENCES key_images(id),
             subaddress_major INTEGER NOT NULL,
-            subaddress_minor INTEGER NOT NULL,
-            FOREIGN KEY(subaddress_major, subaddress_minor) REFERENCES subaddresses(major_index, minor_index)
+            subaddress_minor INTEGER NOT NULL
           );
           CREATE INDEX output_key_image ON outputs(key_image);
 
@@ -126,14 +133,14 @@ namespace wallet
           CREATE TRIGGER output_received AFTER INSERT ON outputs
           FOR EACH ROW
           BEGIN
-            UPDATE metadata SET balance = balance + NEW.amount WHERE id = 0;
+            UPDATE metadata SET val_numeric = val_numeric + NEW.amount WHERE id = 'balance';
           END;
 
           -- update balance when output removed (blockchain re-org)
           CREATE TRIGGER output_removed AFTER DELETE ON outputs
           FOR EACH ROW
           BEGIN
-            UPDATE metadata SET balance = balance - OLD.amount WHERE id = 0;
+            UPDATE metadata SET val_numeric = val_numeric - OLD.amount WHERE id = 'balance';
           END;
 
           CREATE TABLE spends (
@@ -150,7 +157,7 @@ namespace wallet
           FOR EACH ROW
           BEGIN
             UPDATE outputs SET spent_height = NEW.height WHERE key_image = NEW.key_image;
-            UPDATE metadata SET balance = balance - (SELECT outputs.amount FROM outputs WHERE outputs.key_image = NEW.key_image);
+            UPDATE metadata SET val_numeric = val_numeric - (SELECT outputs.amount FROM outputs WHERE outputs.key_image = NEW.key_image) where id = 'balance';
           END;
 
           -- update output and balance when output un-seen as spent (blockchain re-org)
@@ -158,7 +165,7 @@ namespace wallet
           FOR EACH ROW
           BEGIN
             UPDATE outputs SET spent_height = 0 WHERE key_image = OLD.key_image;
-            UPDATE metadata SET balance = balance + (SELECT outputs.amount FROM outputs WHERE outputs.key_image = OLD.key_image);
+            UPDATE metadata SET val_numeric = val_numeric + (SELECT outputs.amount FROM outputs WHERE outputs.key_image = OLD.key_image) where id = 'balance';
           END;
 
           CREATE TRIGGER key_image_output_removed_cleaner AFTER DELETE ON outputs
@@ -176,34 +183,71 @@ namespace wallet
 
         )");
 
-    prepared_exec("UPDATE metadata SET nettype = ? WHERE id = 0;", std::string(cryptonote::network_type_to_string(nettype)));
+    set_metadata_text("nettype", std::string(cryptonote::network_type_to_string(nettype)));
 
     db_tx.commit();
   }
 
+  // Helpers to access the metadata table
+  void
+  WalletDB::set_metadata_int(const std::string& id, int64_t val)
+  {
+    prepared_exec("INSERT INTO metadata(id, val_numeric) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET val_numeric=excluded.val_numeric", id, val);
+  }
+
+  int64_t
+  WalletDB::get_metadata_int(const std::string& id)
+  {
+    return prepared_get<int64_t>("SELECT val_numeric FROM metadata WHERE id = ?", id);
+  }
+
+  void
+  WalletDB::set_metadata_text(const std::string& id, const std::string& val)
+  {
+    prepared_exec("INSERT INTO metadata(id, val_text) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET val_text=excluded.val_text", id, val);
+  }
+
+  std::string
+  WalletDB::get_metadata_text(const std::string& id)
+  {
+    return prepared_get<std::string>("SELECT val_text FROM metadata WHERE id = ?", id);
+  }
+
+  void
+  WalletDB::set_metadata_blob(const std::string& id, std::string_view data)
+  {
+    prepared_exec("INSERT INTO metadata(id, val_binary) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET val_binary=excluded.val_binary", id, db::blob_binder{data});
+  }
+
+  std::string
+  WalletDB::get_metadata_blob(const std::string& id)
+  {
+    return prepared_get<std::string>("SELECT val_binary FROM metadata WHERE id = ?", id);
+  }
+
   cryptonote::network_type
   WalletDB::network_type()
   {
-    return cryptonote::network_type_from_string(prepared_get<std::string>("SELECT nettype FROM metadata WHERE id=0;"));
+    return cryptonote::network_type_from_string(get_metadata_text("nettype"));
   }
 
 
   void
   WalletDB::add_address(int32_t major_index, int32_t minor_index, const std::string& address)
   {
-    auto exists = prepared_get<int64_t>("SELECT COUNT(*) FROM subaddresses WHERE major_index = ? AND minor_index = ?;",
+    auto exists = prepared_get<int64_t>("SELECT COUNT(*) FROM subaddresses WHERE major_index = ? AND minor_index = ?",
         major_index,
         minor_index);
 
     if (exists)
     {
-      auto existing_addr = prepared_get<std::string>("SELECT address FROM subaddresses WHERE major_index = ? AND minor_index = ?;",
+      auto existing_addr = prepared_get<std::string>("SELECT address FROM subaddresses WHERE major_index = ? AND minor_index = ?",
           major_index,
           minor_index);
 
       if (major_index == 0 and minor_index == 0 and existing_addr == "")
       {
-        prepared_exec("UPDATE subaddresses SET address = ? WHERE major_index = ? AND minor_index = ?;",
+        prepared_exec("UPDATE subaddresses SET address = ? WHERE major_index = ? AND minor_index = ?",
             address,
             major_index,
             minor_index);
@@ -216,7 +260,7 @@ namespace wallet
     }
     else
     {
-      prepared_exec("INSERT INTO subaddresses(major_index, minor_index, address, used) VALUES(?,?,?);",
+      prepared_exec("INSERT INTO subaddresses(major_index, minor_index, address, used) VALUES(?,?,?)",
           major_index,
           minor_index,
           address,
@@ -227,7 +271,7 @@ namespace wallet
   std::string
   WalletDB::get_address(int32_t major_index, int32_t minor_index)
   {
-    auto addr = prepared_maybe_get<std::string>("SELECT address FROM subaddresses WHERE major_index = ? AND minor_index = ?;",
+    auto addr = prepared_maybe_get<std::string>("SELECT address FROM subaddresses WHERE major_index = ? AND minor_index = ?",
         major_index,
         minor_index);
 
@@ -337,26 +381,38 @@ namespace wallet
   int64_t
   WalletDB::last_scan_height()
   {
-    return prepared_get<int64_t>("SELECT last_scan_height FROM metadata WHERE id=0;");
+    return get_metadata_int("last_scan_height");
   }
 
   int64_t
   WalletDB::scan_target_height()
   {
-    return prepared_get<int64_t>("SELECT scan_target_height FROM metadata WHERE id=0;");
+    return get_metadata_int("scan_target_height");
+  }
+
+  int64_t
+  WalletDB::current_height()
+  {
+    return prepared_get<int64_t>("SELECT max(height) from blocks");
   }
 
   void
   WalletDB::update_top_block_info(int64_t height, const crypto::hash& hash)
   {
-    prepared_exec("UPDATE metadata SET scan_target_height = ?, scan_target_hash = ? WHERE id = 0",
-      height, tools::type_to_hex(hash));
+    set_metadata_int("scan_target_height", height);
+    set_metadata_text("scan_target_hash", tools::type_to_hex(hash));
   }
 
   int64_t
   WalletDB::overall_balance()
   {
-    return prepared_get<int64_t>("SELECT balance FROM metadata WHERE id=0;");
+    return get_metadata_int("balance");
+  }
+
+  int64_t
+  WalletDB::unlocked_balance()
+  {
+    return prepared_get<int64_t>("SELECT sum(o.amount) FROM outputs AS o WHERE o.spent_height = 0 AND o.spending = false AND (o.block_height + o.unlock_time) <= (SELECT m.val_numeric FROM metadata as m WHERE m.id = 'last_scan_height')");
   }
 
   int64_t
@@ -418,7 +474,46 @@ namespace wallet
   int64_t
   WalletDB::chain_output_count()
   {
-    return prepared_get<int64_t>("SELECT output_count FROM metadata WHERE id=0;");
+    return get_metadata_int("output_count");
+  }
+  void
+  WalletDB::save_keys(const std::shared_ptr<WalletKeys> keys)
+  {
+    const auto maybe_db_keys = load_keys();
+    if (maybe_db_keys.has_value())
+    {
+      if ((tools::view_guts(maybe_db_keys->spend_privkey()) != tools::view_guts(keys->spend_privkey())) ||
+          (tools::view_guts(maybe_db_keys->spend_pubkey()) != tools::view_guts(keys->spend_pubkey())) ||
+          (tools::view_guts(maybe_db_keys->view_privkey()) != tools::view_guts(keys->view_privkey())) ||
+          (tools::view_guts(maybe_db_keys->view_pubkey()) != tools::view_guts(keys->view_pubkey())))
+            throw std::runtime_error("provided keys do not match database file");
+    }
+
+    set_metadata_blob_guts("spend_priv", keys->spend_privkey());
+    set_metadata_blob_guts("spend_pub",  keys->spend_pubkey());
+    set_metadata_blob_guts("view_priv",  keys->view_privkey());
+    set_metadata_blob_guts("view_pub",   keys->view_pubkey());
   }
 
+  std::optional<DBKeys>
+  WalletDB::load_keys()
+  {
+    DBKeys keys;
+    // Will throw if the keys do not exist in the database (for example when the wallet is first created) 
+    // catch this and return nullopt. This means all 4 keys need to exist in the database. In future 
+    // view only wallets will need the ability to return an empty spend_priv key
+    try {
+      keys.ssk = get_metadata_blob_guts<crypto::secret_key>("spend_priv");
+      keys.spk = get_metadata_blob_guts<crypto::public_key>("spend_pub");
+      keys.vsk = get_metadata_blob_guts<crypto::secret_key>("view_priv");
+      keys.vpk = get_metadata_blob_guts<crypto::public_key>("view_pub");
+    }
+    catch (const std::exception& e)
+    {
+      oxen::log::debug(logcat, "Could not load keys: {}", e.what());
+      return std::nullopt;
+    }
+
+    return keys;
+  }
 }  // namespace wallet

src/wallet3/db/walletdb.hpp

@@ -3,7 +3,8 @@
 #include <SQLiteCpp/SQLiteCpp.h>
 #include <sqlitedb/database.hpp>
 
-#include "output.hpp"
+#include "wallet3/output.hpp"
+#include "wallet3/walletkeys.hpp"
 
 #include <optional>
 
@@ -37,6 +38,20 @@ namespace wallet
     void
     create_schema(cryptonote::network_type nettype = cryptonote::network_type::TESTNET);
 
+    // Helpers to access the metadata table
+    void set_metadata_int(const std::string& id, int64_t val);
+    int64_t get_metadata_int(const std::string& id);
+    void set_metadata_text(const std::string& id, const std::string& val);
+    std::string get_metadata_text(const std::string& id);
+    void set_metadata_blob(const std::string& id, std::string_view data);
+    std::string get_metadata_blob(const std::string& id);
+
+    template <typename T>
+    void set_metadata_blob_guts(const std::string& id, const T& val) { set_metadata_blob(id, tools::view_guts(val)); }
+
+    template <typename T>
+    T get_metadata_blob_guts(const std::string& id) { return prepared_get<db::blob_guts<T>>("SELECT val_binary FROM metadata WHERE id = ?", id); };
+
     cryptonote::network_type
     network_type();
 
@@ -70,6 +85,10 @@ namespace wallet
     int64_t
     scan_target_height();
 
+    // Returns the height of the highest block in the database
+    int64_t
+    current_height();
+
     // Update the top block height and hash.
     void
     update_top_block_info(int64_t height, const crypto::hash& hash);
@@ -78,6 +97,10 @@ namespace wallet
     int64_t
     overall_balance();
 
+    // Get unlocked balance across all subaddresses
+    int64_t
+    unlocked_balance();
+
     // Get available balance with amount above an optional minimum amount.
     // TODO: subaddress specification
     int64_t
@@ -92,5 +115,13 @@ namespace wallet
     // and thus mixable, this can be used for decoy selection.
     int64_t
     chain_output_count();
+
+    // Saves keys to the database, will check if keys match if already exists and throw if different
+    void
+    save_keys(const std::shared_ptr<WalletKeys> keys);
+
+    // Loads keys from an already created database
+    std::optional<DBKeys>
+    load_keys();
   };
 }

src/wallet3/default_daemon_comms.cpp

@@ -7,27 +7,26 @@
 
 #include <cryptonote_basic/cryptonote_format_utils.h>
 #include <common/string_util.h>
-#include <epee/misc_log_ex.h>
 
 #include <iostream>
 
 namespace wallet
 {
+  static auto logcat = oxen::log::Cat("wallet");
+
   void
   DefaultDaemonComms::on_get_blocks_response(std::vector<std::string> response)
   {
     if (not response.size())
     {
-      std::cout << "on_get_blocks_response(): empty get_blocks response\n";
-      //TODO: error handling
+      oxen::log::warning(logcat, "on_get_blocks_response(): empty get_blocks response");
       return;
     }
 
     const auto& status = response[0];
     if (status != "OK" and status != "END")
     {
-      std::cout << "get_blocks response: " << response[0] << "\n";
-      //TODO: error handling
+      oxen::log::warning(logcat, "get_blocks response: {}", response[0]);
       return;
     }
 
@@ -35,7 +34,7 @@ namespace wallet
     // TODO: decide/confirm this behavior on the daemon side of things
     if (response.size() == 1)
     {
-      std::cout << "get_blocks response.size() == 1\n";
+      oxen::log::warning(logcat, "get_blocks response.size() == 1");
       return;
     }
 
@@ -103,13 +102,13 @@ namespace wallet
     }
     catch (const std::exception& e)
     {
-      std::cout << e.what() << "\n";
+      oxen::log::warning(logcat, "exception thrown: {}", e.what());
       return;
     }
 
     if (blocks.size() == 0)
     {
-      std::cout << "received no blocks, but server said response OK\n";
+      oxen::log::warning(logcat, "received no blocks, but server said response OK");
       return;
     }
 
@@ -119,7 +118,7 @@ namespace wallet
     omq->job([blocks=std::move(blocks),this](){
         for_each_wallet([&](std::shared_ptr<Wallet> wallet){
             wallet->add_blocks(blocks);
-            });
+          });
         }, sync_thread);
 
     if (status == "END")
@@ -142,6 +141,7 @@ namespace wallet
   void
   DefaultDaemonComms::request_top_block_info()
   {
+    oxen::log::trace(logcat, "request top block called");
     auto timeout_job = [self=weak_from_this()](){
       if (auto comms = self.lock())
         comms->request_top_block_info();
@@ -155,9 +155,11 @@ namespace wallet
     else
       omq->add_timer(status_timer, timeout_job, 15s);
 
+    oxen::log::trace(logcat, "requesting rpc.get_height");
     omq->request(conn, "rpc.get_height",
         [this](bool ok, std::vector<std::string> response)
         {
+          oxen::log::trace(logcat, "rpc get_height response");
           if (not ok or response.size() != 2 or response[0] != "200")
             return;
 
@@ -167,11 +169,17 @@ namespace wallet
           crypto::hash new_hash;
 
           if (not dc.skip_until("hash"))
+          {
+            oxen::log::warning(logcat, "bad response from rpc.get_height, key 'hash' missing");
             throw std::runtime_error("bad response from rpc.get_height, key 'hash' missing");
+          }
           new_hash = tools::make_from_guts<crypto::hash>(dc.consume_string_view());
 
           if (not dc.skip_until("height"))
+          {
+            oxen::log::warning(logcat, "bad response from rpc.get_height, key 'height' missing");
             throw std::runtime_error("bad response from rpc.get_height, key 'height' missing");
+          }
           new_height = dc.consume_integer<int64_t>();
 
           bool got_new = (new_height > (top_block_height + 1));
@@ -193,9 +201,11 @@ namespace wallet
           }
         }, "de");
 
+    oxen::log::trace(logcat, "requesting rpc.get_fee_estimate");
     omq->request(conn, "rpc.get_fee_estimate",
         [this](bool ok, std::vector<std::string> response)
         {
+          oxen::log::trace(logcat, "rpc get_fee estimate response");
           if (not ok or response.size() != 2 or response[0] != "200")
             return;
 
@@ -205,11 +215,17 @@ namespace wallet
           int64_t new_fee_per_output = 0;
 
           if (not dc.skip_until("fee_per_byte"))
+          {
+            oxen::log::warning(logcat, "bad response from rpc.get_fee_estimate, key 'fee_per_byte' missing");
             throw std::runtime_error("bad response from rpc.get_fee_estimate, key 'fee_per_byte' missing");
+          }
           new_fee_per_byte = dc.consume_integer<int64_t>();
 
           if (not dc.skip_until("fee_per_output"))
+          {
+            oxen::log::warning(logcat, "bad response from rpc.get_fee_estimate, key 'fee_per_output' missing");
             throw std::runtime_error("bad response from rpc.get_fee_estimate, key 'fee_per_output' missing");
+          }
           new_fee_per_output = dc.consume_integer<int64_t>();
 
           fee_per_byte = new_fee_per_byte;
@@ -229,6 +245,7 @@ namespace wallet
   void
   DefaultDaemonComms::set_remote(std::string_view address)
   {
+    oxen::log::info(logcat, "Set remote called with address: {}", address);
     try
     {
       remote = oxenmq::address{address};
@@ -239,8 +256,16 @@ namespace wallet
       throw;
     }
 
-    // TODO: proper callbacks
-    conn = omq->connect_remote(remote, [](auto){}, [](auto,auto){});
+    oxen::log::info(logcat, "Trying to connect to remote oxend");
+    conn = omq->connect_remote(remote,
+        // Callback for success case of connect remote
+        [](auto){
+          oxen::log::info(logcat, "successfully connected via OMQ");
+        },
+        // Callback for failure case of connect remote
+        [](auto, auto reason){
+          oxen::log::error(logcat, "Daemon Comms was not successful in connecting to remote oxend. Reason: {}", reason);
+        });
 
     request_top_block_info();
   }
@@ -301,10 +326,9 @@ namespace wallet
       // if not OK
       if (response[0] != "200")
       {
-        std::cout << "get_outputs response not ok: " << response[0] << "\n";
+        oxen::log::warning(logcat, "get_outputs response not ok: {}", response[0]);
         if (response.size() == 2)
-          std::cout << " -- error: \"" << response[1] << "\"\n";
-        //TODO: error handling
+          oxen::log::warning(logcat, " -- error: \"{}\"", response[1]);
         return;
       }
 
@@ -312,7 +336,7 @@ namespace wallet
       // TODO: decide/confirm this behavior on the daemon side of things
       if (response.size() == 1)
       {
-        std::cout << "get_blocks response.size() == 1\n";
+        oxen::log::warning(logcat, "get_blocks response.size() == 1");
         return;
       }
 
@@ -365,13 +389,13 @@ namespace wallet
       }
       catch (const std::exception& e)
       {
-        std::cout << e.what() << "\n";
+        oxen::log::warning(logcat, "exception thrown: {}", e.what());
         return;
       }
 
       if (outputs.size() == 0)
       {
-        std::cout << "received no outputs, but server said response OK\n";
+        oxen::log::warning(logcat, "received no outputs, but server said response OK");
         return;
       }
 
@@ -398,34 +422,27 @@ namespace wallet
     auto fut = p->get_future();
     auto req_cb = [p=std::move(p)](bool ok, std::vector<std::string> response)
     {
-      // TODO: handle various error cases.
-      if (not ok or response.size() != 2 or response[0] != "200")
-      {
-        p->set_value("Unknown Error");
-        return;
-      }
-      else
+      try 
       {
+        if (not ok or response.size() != 2 or response[0] != "200")
+          throw std::runtime_error{"Unknown Error"};
+
         oxenc::bt_dict_consumer dc{response[1]};
         if (dc.skip_until("reason"))
-        {
-          auto reason = dc.consume_string();
-          p->set_value(std::string("Submit Transaction rejected, reason: ") + reason);
-          return;
-        }
+          throw std::runtime_error{"Submit Transaction rejected, reason: " + dc.consume_string()};
 
         if (not dc.skip_until("status"))
-        {
-          p->set_value("Invalid response from daemon");
-          return;
-        }
+          throw std::runtime_error{"Invalid response from daemon"};
 
         auto status = dc.consume_string();
 
-        if (status == "OK")
-          p->set_value("OK");
-        else
-          p->set_value(std::string("Something getting wrong.") + status);
+        if (status != "OK")
+          throw std::runtime_error{"Submit Transaction rejected, reason: " + status};
+
+        p->set_value("OK");
+
+      } catch (...) {
+        p->set_exception(std::current_exception());
       }
     };
 
@@ -443,10 +460,58 @@ namespace wallet
     return fut;
   }
 
+  std::future<std::pair<std::string, crypto::hash>>
+  DefaultDaemonComms::ons_names_to_owners(const std::string& name_hash, const uint16_t type)
+  {
+    auto p = std::make_shared<std::promise<std::pair<std::string, crypto::hash>>>();
+    auto fut = p->get_future();
+    auto req_cb = [p=std::move(p)](bool ok, std::vector<std::string> response)
+    {
+      try 
+      {
+        oxenc::bt_dict_consumer dc{response[1]};
+
+        if (not dc.skip_until("result"))
+          throw std::runtime_error{"Invalid response from daemon"};
+
+        auto result_list = dc.consume_list_consumer();
+        auto result = result_list.consume_dict_consumer();
+
+        crypto::hash prev_txid;
+        std::string curr_owner;
+
+        if (not result.skip_until("owner"))
+          throw std::runtime_error{"Invalid response from daemon"};
+
+        curr_owner = dc.consume_string();
+
+        if (not result.skip_until("txid"))
+          throw std::runtime_error{"Invalid response from daemon"};
+
+        tools::hex_to_type<crypto::hash>(dc.consume_string(), prev_txid);
+
+        p->set_value(std::make_pair(curr_owner, prev_txid));
+      } catch (...) {
+        p->set_exception(std::current_exception());
+      }
+    };
+
+    oxenc::bt_dict req_params_dict{
+      {"name_hash", oxenc::bt_list{{name_hash}}},
+      {"type", oxenc::bt_list{{type}}}
+    };
+
+    omq->request(conn, "rpc.ons_names_to_owners", req_cb, oxenc::bt_serialize(req_params_dict));
+
+    return fut;
+  }
+
   void
   DefaultDaemonComms::register_wallet(wallet::Wallet& wallet, int64_t height, bool check_sync_height, bool new_wallet)
   {
+    oxen::log::trace(logcat, "Daemon Comms register_wallet called");
     omq->job([this,w=wallet.shared_from_this(),height,check_sync_height,new_wallet](){
+        oxen::log::trace(logcat, "register_wallet lambda called");
         if (wallets.count(w))
           wallets[w] = height;
         else if (new_wallet)
@@ -472,6 +537,7 @@ namespace wallet
   void
   DefaultDaemonComms::deregister_wallet(wallet::Wallet& wallet, std::promise<void>& p)
   {
+    oxen::log::trace(logcat, "Daemon Comms deregister_wallet called");
     auto dereg_finish = [this,&p]() mutable {
       p.set_value();
     };
@@ -493,7 +559,7 @@ namespace wallet
             syncing = false;
           }
 
-          std::cout << "deregister_wallet() setting sync_from_height to " << sync_from_height << "\n";
+          oxen::log::debug(logcat, "deregister_wallet() setting sync_from_height to {}", sync_from_height);
           if (sync_from_height != 0 and sync_from_height == top_block_height)
             syncing = false;
         }, sync_thread);
@@ -527,6 +593,7 @@ namespace wallet
     if ((not syncing and sync_from_height <= top_block_height) or (top_block_height == 0))
     {
       syncing = true;
+      oxen::log::debug(logcat, "Start Syncing");
       get_blocks();
     }
   }

src/wallet3/default_daemon_comms.hpp

@@ -57,6 +57,9 @@ namespace wallet
     std::future<std::string>
     submit_transaction(const cryptonote::transaction& tx, bool blink);
 
+    std::future<std::pair<std::string, crypto::hash>>
+    ons_names_to_owners(const std::string& name_hash, const uint16_t type);
+
    private:
 
     void

src/wallet3/keyring.cpp

@@ -13,6 +13,8 @@
 
 namespace wallet
 {
+  static auto logcat = oxen::log::Cat("wallet");
+
   std::string
   Keyring::get_main_address()
   {
@@ -23,7 +25,6 @@ namespace wallet
   crypto::secret_key
   Keyring::generate_tx_key(cryptonote::hf hf_version)
   {
-    // TODO sean make sure this is zero
     crypto::secret_key tx_key{};
 
     if (!key_device.open_tx(tx_key, cryptonote::transaction::get_max_version_for_hf(hf_version), cryptonote::txtype::standard))
@@ -35,7 +36,6 @@ namespace wallet
   crypto::public_key
   Keyring::secret_tx_key_to_public_tx_key(const crypto::secret_key a)
   {
-    // TODO sean make sure this is zero
     rct::key aG{};
     if (!key_device.scalarmultBase(aG, rct::sk2rct(a)))
       throw std::runtime_error("Could not convert secret tx key to public tx key");
@@ -83,11 +83,11 @@ namespace wallet
   {
     auto candidate_key = output_spend_key(derivation, output_key, output_index);
 
-    // TODO: handle checking against subaddresses
-    if (candidate_key == spend_public_key)
-    {
-      return cryptonote::subaddress_index{0, 0};
-    }
+    // Searchs against our map for subaddress public view keys which also includes our
+    // regular view key at index (0,0)
+    if (const auto subaddress_index = subaddresses.find(candidate_key); subaddress_index != subaddresses.end())
+      return subaddress_index->second;
+
     return std::nullopt;
   }
 
@@ -98,13 +98,7 @@ namespace wallet
       uint64_t output_index,
       const cryptonote::subaddress_index& sub_index)
   {
-    // TODO: subaddress support, for now throw if not main address
-    if (not sub_index.is_zero())
-    {
-      throw std::invalid_argument("Subaddresses not yet supported in wallet3");
-    }
-
-    auto output_private_key = derive_transaction_secret_key(derivation, output_index);
+    auto output_private_key = derive_output_secret_key(derivation, output_index, sub_index);
 
     crypto::public_key output_pubkey_computed;
     key_device.secret_key_to_public_key(output_private_key, output_pubkey_computed);
@@ -194,13 +188,18 @@ namespace wallet
 
   // This is called over a transaction input to produce the secret key that can spend an outputs funds.
   // The key derivation is usually produced from calling generate_key_derivation().
-  // computes Hs(a*R || idx) + b
-  // TODO: subaddress support
+  // computes Hs(a*R || idx) + b if main address
+  // computes Hs(a*R || idx) + b + m if subaddress
   crypto::secret_key
-  Keyring::derive_transaction_secret_key(const crypto::key_derivation& key_derivation, const size_t output_index)
+  Keyring::derive_output_secret_key(const crypto::key_derivation& key_derivation, const size_t output_index, const cryptonote::subaddress_index& sub_index)
   {
     crypto::secret_key output_secret_key;
     key_device.derive_secret_key(key_derivation, output_index, spend_private_key, output_secret_key);
+
+    // If we have a subaddress that received the output then add the subaddress private key to the output secret key
+    if (!sub_index.is_zero())
+      key_device.sc_secret_add(output_secret_key, output_secret_key, key_device.get_subaddress_secret_key(view_private_key, sub_index));
+
     return output_secret_key;
   }
 
@@ -252,8 +251,8 @@ namespace wallet
       // blockchain to see if it is ours to spend. We already know its ours because the wallet 
       // has collected them at an earlier point in time. Now we combine this derivation
       // with the output index and our secret spend key to generate
-      // the actual transaction secret key which we can use to spend the output.
-      crypto::secret_key output_secret_key = derive_transaction_secret_key(src_entr.derivation, src_entr.output_index);
+      // the output secret key which we can use to spend the output.
+      crypto::secret_key output_secret_key = derive_output_secret_key(src_entr.derivation, src_entr.output_index, src_entr.subaddress_index);
 
       crypto::public_key computed_output_pubkey{};
       if (!key_device.secret_key_to_public_key(output_secret_key, computed_output_pubkey)
@@ -392,6 +391,57 @@ namespace wallet
       throw std::runtime_error("RCT signing went wrong -- verRctNonSemanticsSimple returned false");
   }
 
+  // Will create subaddress spend public keys from {account, begin} to {account, end} inclusive of begin and end
+  std::vector<crypto::public_key> Keyring::get_subaddress_spend_public_keys(uint32_t account, uint32_t begin, uint32_t end) {
+    if (begin > end)
+      throw std::runtime_error("begin > end");
+
+    std::vector<crypto::public_key> pkeys;
+    pkeys.reserve(end - begin + 1);
+    cryptonote::subaddress_index index = {account, begin};
+
+    ge_p3 p3;
+    ge_cached cached;
+    if (ge_frombytes_vartime(&p3, spend_public_key.data()) != 0)
+      throw std::runtime_error("ge_frombytes_vartime failed to convert spend public key");
+    ge_p3_to_cached(&cached, &p3);
+
+    for (uint32_t idx = begin; idx <= end; ++idx)
+    {
+      index.minor = idx;
+      if (index.is_zero())
+      {
+        pkeys.push_back(spend_public_key);
+        continue;
+      }
+      crypto::secret_key m = key_device.get_subaddress_secret_key(view_private_key, index);
+
+      // M = m*G
+      ge_scalarmult_base(&p3, m.data());
+
+      // D = B + M
+      crypto::public_key D;
+      ge_p1p1 p1p1;
+      ge_add(&p1p1, &p3, &cached);
+      ge_p1p1_to_p3(&p3, &p1p1);
+      ge_p3_tobytes(D.data(), &p3);
+
+      pkeys.push_back(D);
+    }
+    return pkeys;
+  }
+
+  void
+  Keyring::expand_subaddresses(const cryptonote::subaddress_index& lookahead)
+  {
+    for (uint32_t i = 0; i < lookahead.major; i++) {
+      const std::vector<crypto::public_key> pkeys = get_subaddress_spend_public_keys(i, 0, lookahead.minor);
+      for (uint32_t j = 0; j < lookahead.minor; j++) {
+         subaddresses[pkeys[j]] = {i,j};
+      }
+    }
+  }
+
   cryptonote::account_keys
   Keyring::export_keys()
   {
@@ -402,4 +452,36 @@ namespace wallet
     return returned_keys;
   }
 
+  ons::generic_signature
+  Keyring::generate_ons_signature(const std::string& curr_owner, const ons::generic_owner* new_owner, const ons::generic_owner* new_backup_owner, const ons::mapping_value& encrypted_value, const crypto::hash& prev_txid, const cryptonote::network_type& nettype)
+  {
+    ons::generic_signature result;
+    cryptonote::address_parse_info curr_owner_parsed = {};
+    if (!cryptonote::get_account_address_from_str(curr_owner_parsed, nettype, curr_owner))
+        throw std::runtime_error("Could not parse address");
+
+    //TODO sean this should actually get it from the db
+    cryptonote::subaddress_index index = {0,0};
+
+    //std::optional<cryptonote::subaddress_index> index = get_subaddress_index(curr_owner_parsed.address);
+    //if (!index) return false;
+
+    auto sig_data = ons::tx_extra_signature(
+        encrypted_value.to_view(),
+        new_owner,
+        new_backup_owner,
+        prev_txid);
+    if (sig_data.empty())
+      throw std::runtime_error("Could not generate signature");
+
+    cryptonote::account_base account;
+    account.create_from_keys(cryptonote::account_public_address{spend_public_key, view_public_key}, spend_private_key, view_private_key);
+    auto& hwdev = account.get_device();
+    hw::mode_resetter rst{key_device};
+    key_device.generate_ons_signature(sig_data, account.get_keys(), index, result.monero);
+    result.type = ons::generic_owner_sig_type::monero;
+
+    return result;
+  }
+
 }  // namespace wallet

src/wallet3/keyring.hpp

@@ -3,16 +3,18 @@
 #include <crypto/crypto.h>
 #include <cryptonote_basic/subaddress_index.h>
 #include <cryptonote_basic/cryptonote_basic.h>
+#include <cryptonote_core/oxen_name_system.h>
 #include <device/device_default.hpp>
 #include <ringct/rctSigs.h>
 
 #include <optional>
 
 #include "pending_transaction.hpp"
+#include "walletkeys.hpp"
 
 namespace wallet
 {
-  class Keyring
+  class Keyring : public WalletKeys
   {
    public:
     Keyring(
@@ -28,6 +30,20 @@ namespace wallet
         , nettype(_nettype)
     {}
 
+    Keyring(
+        std::string _spend_private_key,
+        std::string _spend_public_key,
+        std::string _view_private_key,
+        std::string _view_public_key,
+        cryptonote::network_type _nettype = cryptonote::network_type::TESTNET)
+        : nettype(_nettype)
+    {
+      tools::hex_to_type<crypto::secret_key>(_spend_private_key, spend_private_key);
+      tools::hex_to_type<crypto::public_key>(_spend_public_key, spend_public_key);
+      tools::hex_to_type<crypto::secret_key>(_view_private_key, view_private_key);
+      tools::hex_to_type<crypto::public_key>(_view_public_key, view_public_key);
+    }
+
     Keyring() {}
 
     virtual std::string
@@ -94,9 +110,10 @@ namespace wallet
         std::vector<rct::key>& amount_keys);
 
     virtual crypto::secret_key
-    derive_transaction_secret_key(
+    derive_output_secret_key(
         const crypto::key_derivation& key_derivation,
-        const size_t output_index
+        const size_t output_index,
+        const cryptonote::subaddress_index& sub_index
     );
 
     virtual crypto::hash
@@ -109,19 +126,35 @@ namespace wallet
         PendingTransaction& ptx
     );
 
+    virtual std::vector<crypto::public_key> get_subaddress_spend_public_keys(uint32_t account, uint32_t begin, uint32_t end);
+
+    virtual void
+    expand_subaddresses(const cryptonote::subaddress_index& lookahead);
+
     virtual cryptonote::account_keys
     export_keys();
 
-   private:
+    virtual ons::generic_signature
+    generate_ons_signature(const std::string& curr_owner, const ons::generic_owner* new_owner, const ons::generic_owner* new_backup_owner, const ons::mapping_value& encrypted_value, const crypto::hash& prev_txid, const cryptonote::network_type& nettype);
+
+    cryptonote::network_type nettype;
+
     crypto::secret_key spend_private_key;
     crypto::public_key spend_public_key;
 
     crypto::secret_key view_private_key;
     crypto::public_key view_public_key;
 
-    cryptonote::network_type nettype;
+    const crypto::secret_key& spend_privkey() const override { return spend_private_key; }  
+    const crypto::public_key& spend_pubkey() const override { return spend_public_key; }  
+    const crypto::secret_key& view_privkey() const override { return view_private_key; }  
+    const crypto::public_key& view_pubkey() const override { return view_public_key; }
+
+   private:
 
     hw::core::device_default key_device;
+    //TODO persist the subaddresses list to the database
+    std::unordered_map<crypto::public_key, cryptonote::subaddress_index> subaddresses;
   };
 
 }  // namespace wallet

src/wallet3/pending_transaction.cpp

@@ -15,7 +15,7 @@ namespace wallet
         throw std::runtime_error("Transaction amounts must be positive");
       sum_recipient_amounts += recipient.amount;
     }
-    if (new_recipients.empty() || sum_recipient_amounts < 0)
+    if (sum_recipient_amounts < 0)
       throw std::runtime_error("Transaction amounts must be positive");
   }
 
@@ -55,8 +55,7 @@ namespace wallet
   int64_t
   PendingTransaction::get_fee(int64_t n_inputs) const
   {
-    // TODO sean add this 
-    int64_t fixed_fee = 0;
+    int64_t fixed_fee = burn_fixed;
     // TODO sean add this 
     int64_t burn_pct = 0;
     int64_t fee_percent = oxen::BLINK_BURN_TX_FEE_PERCENT_V18; // 100%
@@ -121,6 +120,8 @@ namespace wallet
       tx.output_unlock_times.push_back(unlock_time);
     tx.output_unlock_times.push_back(change_unlock_time);
 
+    tx.extra = std::move(extra);
+
     return true;
   }
 

src/wallet3/pending_transaction.hpp

@@ -35,7 +35,11 @@ namespace wallet
     uint64_t fee_per_byte = cryptonote::FEE_PER_BYTE_V13;
     uint64_t fee_per_output = cryptonote::FEE_PER_OUTPUT_V18;
     size_t mixin_count = cryptonote::TX_OUTPUT_DECOYS;
-    size_t extra_size() const {return 0;};
+
+    uint64_t burn_fixed = 0;
+
+    std::vector<uint8_t> extra = {};
+    size_t extra_size() const {return extra.size();};
 
     PendingTransaction() = default;
 

src/wallet3/rpc/command_parser.cpp

@@ -360,12 +360,41 @@ void parse_request(SET_LOG_CATEGORIES& req, rpc_input in) {
 }
 
 void parse_request(ONS_BUY_MAPPING& req, rpc_input in) {
+    get_values(in,
+        "account_index", req.request.account_index,
+        "backup_owner", req.request.backup_owner,
+        "do_not_relay", req.request.do_not_relay,
+        "get_tx_hex", req.request.get_tx_hex,
+        "get_tx_key", req.request.get_tx_key,
+        "get_tx_metadata", req.request.get_tx_metadata,
+        "name", req.request.name,
+        "owner", req.request.owner,
+        "priority", req.request.priority,
+        "subaddr_indices", req.request.subaddr_indices,
+        "type", req.request.type,
+        "value", req.request.value
+        );
 }
 
 void parse_request(ONS_RENEW_MAPPING& req, rpc_input in) {
 }
 
 void parse_request(ONS_UPDATE_MAPPING& req, rpc_input in) {
+    get_values(in,
+        "account_index", req.request.account_index,
+        "backup_owner", req.request.backup_owner,
+        "do_not_relay", req.request.do_not_relay,
+        "get_tx_hex", req.request.get_tx_hex,
+        "get_tx_key", req.request.get_tx_key,
+        "get_tx_metadata", req.request.get_tx_metadata,
+        "name", req.request.name,
+        "owner", req.request.owner,
+        "priority", req.request.priority,
+        "signature", req.request.signature,
+        "subaddr_indices", req.request.subaddr_indices,
+        "type", req.request.type,
+        "value", req.request.value
+        );
 }
 
 void parse_request(ONS_MAKE_UPDATE_SIGNATURE& req, rpc_input in) {

src/wallet3/rpc/commands.h

@@ -121,7 +121,6 @@ namespace wallet::rpc {
     };
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Return the wallet's addresses for an account. Optionally filter for specific set of subaddresses.
   ///
   /// Inputs:
@@ -156,7 +155,6 @@ namespace wallet::rpc {
     };
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Get account and address indexes from a specific (sub)address.
   ///
   /// Inputs:
@@ -176,7 +174,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Create a new address for an account. Optionally, label the new address.
   ///
   /// Inputs:
@@ -203,7 +200,20 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
+  /// Returns the status of the wallet
+  ///
+  /// Inputs: No Inputs
+  ///
+  /// Outputs:
+  ///
+  /// - \p syncing -- True/False if the wallet is still syncing
+  /// - \p sync_height -- Current Height of Wallet
+  /// - \p target_height -- Desired Height of the Wallet
+  struct STATUS : NO_ARGS
+  {
+    static constexpr auto names() { return NAMES("status"); }
+  };
+
   /// Label an address.
   ///
   /// Inputs:
@@ -224,7 +234,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Get all accounts for a wallet. Optionally filter accounts by tag.
   ///
   /// Inputs:
@@ -264,7 +273,6 @@ namespace wallet::rpc {
     };
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   // Create a new account with an optional label.
   //
   // Inputs:
@@ -285,7 +293,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Label an account.
   ///
   /// Inputs:
@@ -306,7 +313,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Get a list of user-defined account tags.
   ///
   /// Inputs: None
@@ -332,7 +338,6 @@ namespace wallet::rpc {
     };
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Apply a filtering tag to a list of accounts.
   ///
   /// Inputs:
@@ -353,7 +358,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Remove filtering tag from a list of accounts.
   ///
   /// Inputs:
@@ -372,7 +376,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Set description for an account tag.
   ///
   /// Inputs:
@@ -393,7 +396,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Returns the wallet's current block height and blockchain immutable height
   ///
   /// Inputs: None
@@ -410,7 +412,6 @@ namespace wallet::rpc {
     struct REQUEST : EMPTY {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Send oxen to a number of recipients. To preview the transaction fee, set do_not_relay to true and get_tx_metadata to true.
   /// Submit the response using the data in get_tx_metadata in the RPC call, relay_tx.
   ///
@@ -460,7 +461,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Same as transfer, but can split into more than one tx if necessary.
   ///
   /// Inputs:
@@ -507,7 +507,6 @@ namespace wallet::rpc {
   };
 
   //TODO: Confirm these parameters and descriptions even make sense...
-  OXEN_RPC_DOC_INTROSPECT
   /// Get the details of an unsigned transaction blob
   ///
   /// Inputs:
@@ -563,7 +562,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Sign a transaction created on a read-only wallet (in cold-signing process).
   ///
   /// Inputs:
@@ -590,7 +588,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Submit a previously signed transaction on a read-only wallet (in cold-signing process).
   ///
   /// Inputs:
@@ -610,7 +607,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Send all dust outputs back to the wallet's, to make them easier to spend (and mix).
   ///
   /// Inputs:
@@ -648,7 +644,6 @@ namespace wallet::rpc {
     };
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Send all unlocked balance to an address.
   ///
   /// Inputs:
@@ -704,7 +699,6 @@ namespace wallet::rpc {
     };
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Send all of a specific unlocked output to an address.
   ///
   /// Inputs:
@@ -749,7 +743,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Relay transaction metadata to the daemon
   ///
   /// Inputs:
@@ -771,7 +764,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Tell the wallet to store its data to disk, if needed.
   ///
   /// Inputs: None
@@ -786,7 +778,6 @@ namespace wallet::rpc {
     struct REQUEST : EMPTY {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Payment details struct
   ///
   /// - \p payment_id -- Payment ID matching the input parameter.
@@ -809,7 +800,6 @@ namespace wallet::rpc {
     std::string address;                        // Address receiving the payment.
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Get a list of incoming payments using a given payment id.
   ///
   /// Inputs:
@@ -830,7 +820,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Get a list of incoming payments using a given payment id,
   /// or a list of payments ids, from a given height.
   ///
@@ -858,7 +847,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Transfer details struct
   ///
   /// - \p amount -- Amount of this transfer.
@@ -883,7 +871,6 @@ namespace wallet::rpc {
     bool unlocked;                              // If the TX is spendable yet
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Return a list of incoming transfers to the wallet.
   ///
   /// Inputs:
@@ -908,7 +895,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Return the private view key.
   ///
   /// Inputs: None
@@ -924,7 +910,6 @@ namespace wallet::rpc {
     struct REQUEST : EMPTY {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Return the private spend key.
   ///
   /// Inputs: None
@@ -940,7 +925,6 @@ namespace wallet::rpc {
     struct REQUEST : EMPTY {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Return the mnemonic.
   ///
   /// Inputs:
@@ -960,7 +944,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Make an integrated address from the wallet address and a payment id.
   ///
   /// Inputs:
@@ -983,7 +966,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Retrieve the standard address and payment id corresponding to an integrated address.
   ///
   /// Inputs:
@@ -1005,7 +987,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   // Stops the wallet, storing the current state.
   //
   // Inputs: None
@@ -1020,7 +1001,6 @@ namespace wallet::rpc {
     struct REQUEST : EMPTY {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Rescan the blockchain from scratch, losing any information
   /// which can not be recovered from the blockchain itself.
   /// This includes destination addresses, tx secret keys, tx notes, etc.
@@ -1042,7 +1022,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Set arbitrary string notes for transactions.
   ///
   /// Inputs:
@@ -1062,7 +1041,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Get string notes for transactions.
   ///
   /// Inputs:
@@ -1082,7 +1060,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Set arbitrary attribute.
   ///
   /// Inputs:
@@ -1102,7 +1079,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Get attribute value by name.
   ///
   /// Inputs:
@@ -1123,7 +1099,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Get transaction secret key from transaction id.
   ///
   /// Inputs:
@@ -1143,7 +1118,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Check a transaction in the blockchain with its secret key.
   ///
   /// Inputs:
@@ -1169,7 +1143,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Get transaction signature to prove it.
   ///
   /// Inputs:
@@ -1193,7 +1166,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Prove a transaction by checking its signature.
   ///
   /// Inputs:
@@ -1222,7 +1194,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Generate a signature to prove a spend. Unlike proving a transaction, it does not requires the destination public address.
   ///
   /// Inputs:
@@ -1244,7 +1215,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Prove a spend using a signature. Unlike proving a transaction, it does not requires the destination public address.
   ///
   /// Inputs:
@@ -1268,7 +1238,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Generate a signature to prove of an available amount in a wallet.
   ///
   /// Inputs:
@@ -1294,7 +1263,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Proves a wallet has a disposable reserve using a signature.
   ///
   /// Inputs:
@@ -1320,7 +1288,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Returns a list of transfers, by default all transfer types are included. If all requested type fields are false, then all transfers will be queried.
   ///
   /// Inputs:
@@ -1374,7 +1341,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Returns a string with the transfers formatted as csv
   ///
   /// Inputs:
@@ -1403,7 +1369,6 @@ namespace wallet::rpc {
     struct REQUEST : GET_TRANSFERS::REQUEST {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Show information about a transfer to/from this address.
   ///
   /// Inputs:
@@ -1428,7 +1393,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Sign a string.
   ///
   /// Inputs:
@@ -1452,7 +1416,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Verify a signature on a string.
   ///
   /// Inputs:
@@ -1476,7 +1439,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Export all outputs in hex format.
   ///
   /// Inputs:
@@ -1496,7 +1458,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Export transfers to csv
   ///
   /// Inputs:
@@ -1540,7 +1501,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Import outputs in hex format.
   ///
   /// Inputs:
@@ -1560,7 +1520,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Export a signed set of key images.
   ///
   /// Inputs:
@@ -1589,7 +1548,6 @@ namespace wallet::rpc {
     };
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Import signed key images list and verify their spent status.
   ///
   /// Inputs:
@@ -1621,7 +1579,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// URI struct
   ///
   /// - \p address -- Wallet address.
@@ -1638,7 +1595,6 @@ namespace wallet::rpc {
     std::string recipient_name; // (Optional) name of the payment recipient.
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Create a payment URI using the official URI spec.
   ///
   /// Inputs:
@@ -1655,7 +1611,6 @@ namespace wallet::rpc {
     struct REQUEST: public uri_spec {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Parse a payment URI to get payment information.
   ///
   /// Inputs:
@@ -1677,7 +1632,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Add an entry to the address book.
   ///
   /// Inputs:
@@ -1699,7 +1653,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Edit a entry in the address book.
   ///
   /// Inputs:
@@ -1725,7 +1678,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Retrieves entries from the address book.
   ///
   /// Inputs:
@@ -1755,7 +1707,6 @@ namespace wallet::rpc {
     };
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Delete an entry from the address book.
   ///
   /// Inputs:
@@ -1773,7 +1724,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Rescan the blockchain for spent outputs.
   ///
   /// Inputs: None
@@ -1786,7 +1736,6 @@ namespace wallet::rpc {
     struct REQUEST : EMPTY {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Refresh a wallet after opening.
   ///
   /// Inputs:
@@ -1807,7 +1756,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Set wallet to (not) auto-refresh on an interval
   ///
   /// Inputs:
@@ -1827,7 +1775,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Start mining in the oxen daemon.
   ///
   /// Inputs:
@@ -1845,7 +1792,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Stop mining in the oxen daemon.
   ///
   /// Inputs: None
@@ -1858,7 +1804,6 @@ namespace wallet::rpc {
     struct REQUEST : EMPTY {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Get a list of available languages for your wallet's seed.
   ///
   /// Inputs: None
@@ -1874,7 +1819,6 @@ namespace wallet::rpc {
     struct REQUEST : EMPTY {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Create a new wallet. You need to have set the argument "'--wallet-dir" when launching oxen-wallet-rpc to make this work.
   ///
   /// Inputs:
@@ -1902,7 +1846,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Open a wallet. You need to have set the argument "--wallet-dir" when launching oxen-wallet-rpc to make this work.
   /// The wallet rpc executable may only open wallet files within the same directory as wallet-dir, otherwise use the
   /// "--wallet-file" flag to open specific wallets.
@@ -1926,7 +1869,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Close the currently opened wallet, after trying to save it.
   ///
   /// Inputs:
@@ -1944,7 +1886,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Change a wallet password.
   ///
   /// Inputs:
@@ -1964,7 +1905,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Restore a wallet using the private spend key, view key and public address.
   ///
   /// Inputs:
@@ -1997,7 +1937,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Restore a wallet using the seed words.
   ///
   /// Inputs:
@@ -2032,7 +1971,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Check if a wallet is a multisig one.
   ///
   /// Inputs: None
@@ -2050,7 +1988,6 @@ namespace wallet::rpc {
     struct REQUEST : EMPTY {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Prepare a wallet for multisig by generating a multisig string to share with peers.
   ///
   /// Inputs: None
@@ -2065,7 +2002,6 @@ namespace wallet::rpc {
     struct REQUEST : EMPTY {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Make a wallet multisig by importing peers multisig string.
   ///
   /// Inputs:
@@ -2090,7 +2026,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Export multisig info for other participants.
   ///
   /// Inputs: None
@@ -2105,7 +2040,6 @@ namespace wallet::rpc {
     struct REQUEST : EMPTY {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Import multisig info from other participants.
   ///
   /// Inputs:
@@ -2125,7 +2059,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Turn this wallet into a multisig wallet, extra step for N-1/N wallets.
   ///
   /// Inputs:
@@ -2147,7 +2080,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// TODO: description
   ///
   /// Inputs:
@@ -2170,7 +2102,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Sign a transaction in multisig.
   ///
   /// Inputs:
@@ -2191,7 +2122,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Submit a signed multisig transaction.
   ///
   /// Inputs:
@@ -2211,7 +2141,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Get RPC version Major & Minor integer-format, where Major is the first 16 bits and Minor the last 16 bits.
   ///
   /// Inputs: None
@@ -2226,7 +2155,6 @@ namespace wallet::rpc {
     struct REQUEST : EMPTY {} request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Stake for Service Node.
   ///
   /// Inputs:
@@ -2269,7 +2197,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Register Service Node.
   ///
   /// Inputs:
@@ -2304,7 +2231,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Request to unlock stake by deregistering Service Node.
   ///
   /// Inputs:
@@ -2325,7 +2251,6 @@ namespace wallet::rpc {
     };
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Check if Service Node can unlock its stake.
   ///
   /// Inputs:
@@ -2346,7 +2271,6 @@ namespace wallet::rpc {
     };
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Parse an address to validate if it's a valid Loki address.
   ///
   /// Inputs:
@@ -2374,7 +2298,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// TODO: description
   ///
   /// Inputs:
@@ -2404,7 +2327,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// TODO: description
   ///
   /// Inputs:
@@ -2422,7 +2344,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// TODO: description
   ///
   /// Inputs:
@@ -2442,7 +2363,6 @@ namespace wallet::rpc {
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Buy a Loki Name System (ONS) mapping that maps a unique name to a Session ID or Lokinet address.
   ///
   /// Currently supports Session, Lokinet and Wallet registrations. Lokinet registrations can be for 1, 2, 5, or 10 years by specifying a type value of "lokinet", "lokinet_2y", "lokinet_5y", "lokinet_10y". Session registrations do not expire.
@@ -2481,37 +2401,34 @@ namespace wallet::rpc {
     static constexpr auto names() { return NAMES("ons_buy_mapping"); }
 
     static constexpr const char *description =
-R"(Buy a Loki Name System (ONS) mapping that maps a unique name to a Session ID or Lokinet address.
+R"(Buy an Oxen Name System (ONS) mapping that maps a unique name to a Session ID, Oxen Address or Lokinet address.
 
-Currently supports Session, Lokinet and Wallet registrations. Lokinet registrations can be for 1, 2, 5, or 10 years by specifying a type value of "lokinet", "lokinet_2y", "lokinet_5y", "lokinet_10y". Session registrations do not expire.
+Currently supports Session, Wallet and Lokinet registrations. Lokinet registrations can be for 1, 2, 5, or 10 years by specifying a type value of "lokinet", "lokinet_2y", "lokinet_5y", "lokinet_10y". Session and Wallet registrations do not expire.
 
 The owner of the ONS entry (by default, the purchasing wallet) will be permitted to submit ONS update transactions to the Loki blockchain (for example to update a Session pubkey or the target Lokinet address). You may change the primary owner or add a backup owner in the registration and can change them later with update transactions. Owner addresses can be either Loki wallets, or generic ed25519 pubkeys (for advanced uses).
 
-For Session, the recommended owner or backup owner is the ed25519 public key of the user's Session ID.
-
 When specifying owners, either a wallet (sub)address or standard ed25519 public key is supported per mapping. Updating the value that a name maps to requires one of the owners to sign the update transaction. For wallets, this is signed using the (sub)address's spend key.
 
 For more information on updating and signing see the ONS_UPDATE_MAPPING documentation.)";
 
     struct REQUEST
     {
-      std::string        type;            // The mapping type: "session", "lokinet", "lokinet_2y", "lokinet_5y", "lokinet_10y", "wallet".
-      std::string        owner;           // (Optional): The ed25519 public key or wallet address that has authority to update the mapping.
-      std::string        backup_owner;    // (Optional): The secondary, backup public key that has authority to update the mapping.
-      std::string        name;            // The name to purchase via Oxen Name Service
-      std::string        value;           // The value that the name maps to via Oxen Name Service, (i.e. For Session: [display name->session public key],  for wallets: [name->wallet address], for Lokinet: [name->domain name]).
+      std::string        type;                 // The mapping type: "session", "wallet", "lokinet", "lokinet_2y", "lokinet_5y", "lokinet_10y".
+      std::optional<std::string> owner;        // (Optional): The ed25519 public key or wallet address that has authority to update the mapping.
+      std::optional<std::string> backup_owner; // (Optional): The secondary, backup public key that has authority to update the mapping.
+      std::string        name;                 // The name to purchase via Oxen Name Service
+      std::string        value;                // The value that the name maps to via Oxen Name Service, (i.e. For Session: [display name->session public key],  for wallets: [name->wallet address], for Lokinet: [name->domain name]).
 
-      uint32_t           account_index;   // (Optional) Transfer from this account index. (Defaults to 0)
-      std::set<uint32_t> subaddr_indices; // (Optional) Transfer from this set of subaddresses. (Defaults to 0)
-      uint32_t           priority;        // Set a priority for the transaction. Accepted values are: or 0-4 for: default, unimportant, normal, elevated, priority.
-      bool               get_tx_key;      // (Optional) Return the transaction key after sending.
-      bool               do_not_relay;    // (Optional) If true, the newly created transaction will not be relayed to the oxen network. (Defaults to false)
-      bool               get_tx_hex;      // Return the transaction as hex string after sending (Defaults to false)
-      bool               get_tx_metadata; // Return the metadata needed to relay the transaction. (Defaults to false)
+      uint32_t           account_index;        // (Optional) Transfer from this account index. (Defaults to 0)
+      std::vector<uint32_t> subaddr_indices;   // (Optional) Transfer from this set of subaddresses. (Defaults to 0)
+      uint32_t           priority;             // Set a priority for the transaction. Accepted values are: or 0-4 for: default, unimportant, normal, elevated, priority.
+      bool               get_tx_key;           // (Optional) Return the transaction key after sending.
+      bool               do_not_relay;         // (Optional) If true, the newly created transaction will not be relayed to the oxen network. (Defaults to false)
+      bool               get_tx_hex;           // Return the transaction as hex string after sending (Defaults to false)
+      bool               get_tx_metadata;      // Return the metadata needed to relay the transaction. (Defaults to false)
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Renew an active lokinet ONS registration
   ///
   /// Renews a Loki Name System lokinet mapping by adding to the existing expiry time.
@@ -2563,7 +2480,6 @@ The renewal can be for 1, 2, 5, or 10 years by specifying a `type` value of "lok
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Update the underlying value in the name->value mapping via Loki Name Service.
   ///
   /// At least one field (value, owner, or backup owner) must be specified in the update.
@@ -2613,13 +2529,13 @@ If signing is performed externally then you must first encrypt the `value` (if b
     {
       std::string        type;      // The mapping type, "session", "lokinet", or "wallet".
       std::string        name;      // The name to update via Loki Name Service
-      std::string        value;     // (Optional): The new value that the name maps to via Loki Name Service. If not specified or given the empty string "", then the mapping's value remains unchanged. If using a `signature` then this value (if non-empty) must be already encrypted.
-      std::string        owner;     // (Optional): The new owner of the mapping. If not specified or given the empty string "", then the mapping's owner remains unchanged.
-      std::string        backup_owner; // (Optional): The new backup owner of the mapping. If not specified or given the empty string "", then the mapping's backup owner remains unchanged.
+      std::optional<std::string> value;     // (Optional): The new value that the name maps to via Loki Name Service. If not specified or given the empty string "", then the mapping's value remains unchanged. If using a `signature` then this value (if non-empty) must be already encrypted.
+      std::optional<std::string> owner; // (Optional): The new owner of the mapping. If not specified or given the empty string "", then the mapping's owner remains unchanged.
+      std::optional<std::string> backup_owner; // (Optional): The new backup owner of the mapping. If not specified or given the empty string "", then the mapping's backup owner remains unchanged.
       std::string        signature; // (Optional): Signature derived using libsodium generichash on {current txid blob, new value blob} of the mapping to update. By default the hash is signed using the wallet's spend key as an ed25519 keypair, if signature is specified.
 
       uint32_t           account_index;    // (Optional) Transfer from this account index. (Defaults to 0)
-      std::set<uint32_t> subaddr_indices;  // (Optional) Transfer from this set of subaddresses. (Defaults to 0)
+      std::vector<uint32_t> subaddr_indices;  // (Optional) Transfer from this set of subaddresses. (Defaults to 0)
       uint32_t           priority;         // Set a priority for the transaction. Accepted values are: 0-4 for: default, unimportant, normal, elevated, priority.
       bool               get_tx_key;       // (Optional) Return the transaction key after sending.
       bool               do_not_relay;     // (Optional) If true, the newly created transaction will not be relayed to the oxen network. (Defaults to false)
@@ -2629,7 +2545,6 @@ If signing is performed externally then you must first encrypt the `value` (if b
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Generate the signature necessary for updating the requested record using the wallet's active [sub]address's spend key. The signature is only valid if the queried wallet is one of the owners of the ONS record.
   ///
   /// This command is only required if the open wallet is one of the owners of a ONS record but wants the update transaction to occur via another non-owning wallet. By default, if no signature is specified to the update transaction, the open wallet is assumed the owner and it's active [sub]address's spend key will automatically be used.
@@ -2666,7 +2581,6 @@ This command is only required if the open wallet is one of the owners of a ONS r
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Takes a ONS name, upon validating it, generates the hash and returns the base64 representation of the hash suitable for use in the daemon ONS name queries.
   ///
   /// Inputs:
@@ -2688,7 +2602,6 @@ This command is only required if the open wallet is one of the owners of a ONS r
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Returns a list of known, plain-text ONS names along with record details for names that this
   /// wallet knows about.  This can optionally decrypt the ONS value as well, or else just return the
   /// encrypted value.
@@ -2736,7 +2649,6 @@ This command is only required if the open wallet is one of the owners of a ONS r
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Adds one or more names to the persistent ONS wallet cache of known names (i.e. for names that
   /// are owned by this wallet that aren't currently in the cache).
   ///
@@ -2763,7 +2675,6 @@ This command is only required if the open wallet is one of the owners of a ONS r
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Takes a ONS encrypted value and encrypts the mapping value using the ONS name.
   ///
   /// Inputs:
@@ -2787,7 +2698,6 @@ This command is only required if the open wallet is one of the owners of a ONS r
     } request;
   };
 
-  OXEN_RPC_DOC_INTROSPECT
   /// Takes a ONS encrypted value and decrypts the mapping value using the ONS name.
   ///
   /// Inputs:
@@ -2916,7 +2826,8 @@ This command is only required if the open wallet is one of the owners of a ONS r
     ONS_KNOWN_NAMES,
     ONS_ADD_KNOWN_NAMES,
     ONS_DECRYPT_VALUE,
-    ONS_ENCRYPT_VALUE
+    ONS_ENCRYPT_VALUE,
+    STATUS
   >;
 
 }

src/wallet3/rpc/omq_server.cpp

@@ -34,9 +34,9 @@ OmqServer::set_omq(std::shared_ptr<oxenmq::OxenMQ> omq_in, wallet::rpc::Config c
 {
   omq = omq_in;
 
-  //TODO: parametrize listening address(es) and auth
-  omq->listen_plain(std::string("ipc://./") + config.sockname);
+  omq->listen_plain("ipc://"s + config.sockname);
 
+  //TODO: parametrize auth
   omq->add_category("rpc", AuthLevel::none, 0 /*no reserved threads*/, 100 /*max queued requests*/);
   // TODO: actually make restricted category require auth
   omq->add_category("restricted", AuthLevel::none, 0 /*no reserved threads*/, 100 /*max queued requests*/);

src/wallet3/rpc/request_handler.cpp

@@ -3,7 +3,7 @@
 #include "commands.h"
 #include "command_parser.h"
 #include <wallet3/wallet.hpp>
-#include <wallet3/db_schema.hpp>
+#include <wallet3/db/walletdb.hpp>
 #include <version.h>
 
 #include <cryptonote_core/cryptonote_tx_utils.h>
@@ -12,6 +12,7 @@
 #include <memory>
 
 #include <common/hex.h>
+#include <oxenc/base64.h>
 #include "mnemonics/electrum-words.h"
 
 
@@ -24,6 +25,8 @@ using cryptonote::rpc::rpc_error;
 
 namespace {
 
+  static auto logcat = oxen::log::Cat("wallet");
+
   template <typename RPC>
   void register_rpc_command(std::unordered_map<std::string, std::shared_ptr<const rpc_command>>& regs)
   {
@@ -56,6 +59,23 @@ void RequestHandler::set_wallet(std::weak_ptr<wallet::Wallet> ptr)
   wallet = ptr;
 }
 
+//TODO sean something here
+std::string RequestHandler::submit_transaction(wallet::PendingTransaction& ptx)
+{
+  std::string response;
+  if (auto w = wallet.lock())
+  {
+    w->keys->sign_transaction(ptx);
+
+    auto submit_future = w->daemon_comms->submit_transaction(ptx.tx, false);
+
+    if (submit_future.wait_for(5s) != std::future_status::ready)
+      throw rpc_error(500, "request to daemon timed out");
+    response = submit_future.get();
+  }
+  return response;
+}
+
 const std::unordered_map<std::string, std::shared_ptr<const rpc_command>> rpc_commands = register_rpc_commands(wallet_rpc_types{});
 
 void RequestHandler::invoke(GET_BALANCE& command, rpc_context context) {
@@ -69,34 +89,32 @@ void RequestHandler::invoke(GET_BALANCE& command, rpc_context context) {
 
 void RequestHandler::invoke(GET_ADDRESS& command, rpc_context context) {
   //TODO: implement fetching address/subaddress from db/keyring
-  /*
   if (auto w = wallet.lock())
   {
-    auto& addresses = (command.response["addresses"] = json::array());
-    if (command.request.address_index.size() == 0)
-    {
-      auto address = w->get_subaddress(command.request.account_index, 0);
-      addresses.push_back(json{
-        {"address", address.as_str(cryptonote::network_type::MAINNET)},
-        {"label", ""},
-        {"address_index", command.request.address_index},
-        {"used", true}
-      });
-    } else {
-      for (const auto& address_index: command.request.address_index)
-      {
-        auto address = w->get_subaddress(command.request.account_index, address_index);
-        addresses.push_back(json{
-          {"address", address.as_str(cryptonote::network_type::MAINNET)},
-          {"label", ""},
-          {"address_index", command.request.address_index},
-          {"used", true}
-        });
-      }
-    }
+    command.response["address"] = w->keys->get_main_address();
+    //auto& addresses = (command.response["addresses"] = json::array());
+    //if (command.request.address_index.size() == 0)
+    //{
+      //auto address = w->get_subaddress(command.request.account_index, 0);
+      //addresses.push_back(json{
+        //{"address", address.as_str(cryptonote::network_type::MAINNET)},
+        //{"label", ""},
+        //{"address_index", command.request.address_index},
+        //{"used", true}
+      //});
+    //} else {
+      //for (const auto& address_index: command.request.address_index)
+      //{
+        //auto address = w->get_subaddress(command.request.account_index, address_index);
+        //addresses.push_back(json{
+          //{"address", address.as_str(cryptonote::network_type::MAINNET)},
+          //{"label", ""},
+          //{"address_index", command.request.address_index},
+          //{"used", true}
+        //});
+      //}
+    //}
   }
-  */
-  
 }
 
 void RequestHandler::invoke(GET_ADDRESS_INDEX& command, rpc_context context) {
@@ -132,16 +150,18 @@ void RequestHandler::invoke(SET_ACCOUNT_TAG_DESCRIPTION& command, rpc_context co
 void RequestHandler::invoke(GET_HEIGHT& command, rpc_context context) {
   if (auto w = wallet.lock())
   {
-    auto height = w->db->scan_target_height();
+    const auto immutable_height = w->db->scan_target_height();
+    const auto height = w->db->current_height();
+
     command.response["height"] = height;
 
-    //TODO: this
-    command.response["immutable_height"] = height;
+    command.response["immutable_height"] = immutable_height;
   }
 }
 
 void RequestHandler::invoke(TRANSFER& command, rpc_context context) {
-std::cout << "rpc handler, handling TRANSFER\n";
+  oxen::log::info(logcat, "RPC Handler received TRANSFER command");
+  wallet::PendingTransaction ptx;
   if (auto w = wallet.lock())
   {
     // TODO: arg checking
@@ -150,8 +170,6 @@ std::cout << "rpc handler, handling TRANSFER\n";
     std::vector<cryptonote::tx_destination_entry> recipients;
     for (const auto& [dest, amount] : command.request.destinations)
     {
-std::cout << "transfer dest: " << dest << "\n";
-std::cout << "transfer amount: " << amount << "\n";
       auto& entry = recipients.emplace_back();
       cryptonote::address_parse_info addr_info;
 
@@ -176,22 +194,10 @@ std::cout << "transfer amount: " << amount << "\n";
     change_dest.is_subaddress = change_addr_info.is_subaddress;
     change_dest.is_integrated = change_addr_info.has_payment_id;
 
-    auto ptx = w->tx_constructor->create_transaction(recipients, change_dest);
-
-    w->keys->sign_transaction(ptx);
-
-std::cout << "rpc, transaction vout.size() = " << ptx.tx.vout.size() << "\n";
-std::cout << "rpc, transaction output_unlock_times.size() = " << ptx.tx.output_unlock_times.size() << "\n";
-std::cout << "rpc, ptx recipients.size() = " << ptx.recipients.size() << "\n";
-
-    auto submit_future = w->daemon_comms->submit_transaction(ptx.tx, false);
-
-    if (submit_future.wait_for(5s) != std::future_status::ready)
-      throw rpc_error(500, "request to daemon timed out");
-
-    command.response["status"] = "200";
-    command.response["result"] = submit_future.get();
+    ptx = w->tx_constructor->create_transaction(recipients, change_dest);
   }
+  command.response["result"] = submit_transaction(ptx);
+  command.response["status"] = "200";
 }
 
 void RequestHandler::invoke(TRANSFER_SPLIT& command, rpc_context context) {
@@ -471,12 +477,69 @@ void RequestHandler::invoke(SET_LOG_CATEGORIES& command, rpc_context context) {
 }
 
 void RequestHandler::invoke(ONS_BUY_MAPPING& command, rpc_context context) {
+  //TODO sean these params need to be accounted for
+  //  "do_not_relay", req.request.do_not_relay.
+  //  "get_tx_hex", req.request.get_tx_hex.
+  //  "get_tx_key", req.request.get_tx_key.
+  //  "get_tx_metadata", req.request.get_tx_metadata.
+  //  "priority", req.request.priority,
+  //  "subaddr_indices", req.request.subaddr_indices,
+
+  oxen::log::info(logcat, "RPC Handler received ONS_BUY_MAPPING command");
+  wallet::PendingTransaction ptx;
+  if (auto w = wallet.lock())
+  {
+    cryptonote::tx_destination_entry change_dest;
+    change_dest.original = w->keys->get_main_address();
+    cryptonote::address_parse_info change_addr_info;
+    cryptonote::get_account_address_from_str(change_addr_info, w->nettype, change_dest.original);
+    change_dest.amount = 0;
+    change_dest.addr = change_addr_info.address;
+    change_dest.is_subaddress = change_addr_info.is_subaddress;
+    change_dest.is_integrated = change_addr_info.has_payment_id;
+
+    ptx = w->tx_constructor->create_ons_buy_transaction(
+      command.request.name,
+      command.request.type,
+      command.request.value,
+      command.request.owner,
+      command.request.backup_owner,
+      change_dest
+      );
+  }
+  command.response["result"] = submit_transaction(ptx);
+  command.response["status"] = "200";
 }
 
 void RequestHandler::invoke(ONS_RENEW_MAPPING& command, rpc_context context) {
 }
 
 void RequestHandler::invoke(ONS_UPDATE_MAPPING& command, rpc_context context) {
+  oxen::log::info(logcat, "RPC Handler received ONS_UPDATE_MAPPING command");
+  wallet::PendingTransaction ptx;
+  if (auto w = wallet.lock())
+  {
+    cryptonote::tx_destination_entry change_dest;
+    change_dest.original = w->keys->get_main_address();
+    cryptonote::address_parse_info change_addr_info;
+    cryptonote::get_account_address_from_str(change_addr_info, w->nettype, change_dest.original);
+    change_dest.amount = 0;
+    change_dest.addr = change_addr_info.address;
+    change_dest.is_subaddress = change_addr_info.is_subaddress;
+    change_dest.is_integrated = change_addr_info.has_payment_id;
+
+    ptx = w->tx_constructor->create_ons_update_transaction(
+      command.request.name,
+      command.request.type,
+      command.request.value,
+      command.request.owner,
+      command.request.backup_owner,
+      change_dest,
+      w->keys
+      );
+  }
+  command.response["result"] = submit_transaction(ptx);
+  command.response["status"] = "200";
 }
 
 void RequestHandler::invoke(ONS_MAKE_UPDATE_SIGNATURE& command, rpc_context context) {
@@ -497,5 +560,18 @@ void RequestHandler::invoke(ONS_ENCRYPT_VALUE& command, rpc_context context) {
 void RequestHandler::invoke(ONS_DECRYPT_VALUE& command, rpc_context context) {
 }
 
+void RequestHandler::invoke(STATUS& command, rpc_context context) {
+  if (auto w = wallet.lock())
+  {
+    const auto sync_height = w->db->current_height();
+    const auto target_height = w->db->scan_target_height();
+
+    command.response["sync_height"] = sync_height;
+    command.response["target_height"] = target_height;
+
+    command.response["syncing"] = sync_height < target_height;
+  }
+}
+
 
 } // namespace wallet::rpc

src/wallet3/rpc/request_handler.h

@@ -3,6 +3,7 @@
 #include "commands.h"
 
 #include "rpc/common/rpc_command.h"
+#include <wallet3/pending_transaction.hpp>
 
 #include <nlohmann/json.hpp>
 #include <oxenc/bt_value.h>
@@ -15,7 +16,6 @@
 namespace wallet {
   class Wallet;
 }
-
 namespace wallet::rpc {
 
 class RequestHandler;
@@ -42,8 +42,12 @@ class RequestHandler {
 
 public:
 
+
+
   void set_wallet(std::weak_ptr<wallet::Wallet> wallet);
 
+  std::string submit_transaction(wallet::PendingTransaction& ptx);
+
   void invoke(GET_BALANCE& command, rpc_context context);
   void invoke(GET_ADDRESS& command, rpc_context context);
   void invoke(GET_ADDRESS_INDEX& command, rpc_context context);
@@ -144,6 +148,7 @@ public:
   void invoke(ONS_ADD_KNOWN_NAMES& command, rpc_context context);
   void invoke(ONS_ENCRYPT_VALUE& command, rpc_context context);
   void invoke(ONS_DECRYPT_VALUE& command, rpc_context context);
+  void invoke(STATUS& command, rpc_context context);
 
 };
 

src/wallet3/transaction_constructor.cpp

@@ -3,7 +3,9 @@
 #include "decoy.hpp"
 #include "output_selection/output_selection.hpp"
 #include "decoy_selection/decoy_selection.hpp"
-#include "db_schema.hpp"
+#include "db/walletdb.hpp"
+
+#include <oxenc/base64.h>
 
 #include <cryptonote_basic/hardfork.h>
 
@@ -20,8 +22,8 @@ namespace wallet
     PendingTransaction new_tx(recipients);
     auto [hf, hf_uint8] = cryptonote::get_ideal_block_version(db->network_type(), db->scan_target_height());
     cryptonote::oxen_construct_tx_params tx_params{hf, cryptonote::txtype::standard, 0, 0};
-    new_tx.tx.version = cryptonote::transaction::get_max_version_for_hf(tx_params.hf_version);
-    new_tx.tx.type = tx_params.tx_type;
+    new_tx.tx.version = cryptonote::transaction::get_max_version_for_hf(hf);
+    new_tx.tx.type = cryptonote::txtype::standard;
     new_tx.fee_per_byte = fee_per_byte;
     new_tx.fee_per_output = fee_per_output;
     new_tx.change = change_recipient;
@@ -29,6 +31,162 @@ namespace wallet
     return new_tx;
   }
 
+  PendingTransaction
+  TransactionConstructor::create_ons_buy_transaction(
+      std::string_view name,
+      std::string_view type_str,
+      std::string_view value,
+      std::optional<std::string_view> owner_str,
+      std::optional<std::string_view> backup_owner_str,
+      const cryptonote::tx_destination_entry& change_recipient
+      )
+  {
+    std::vector<cryptonote::tx_destination_entry> recipients;
+    PendingTransaction new_tx(recipients);
+    auto [hf, hf_uint8] = cryptonote::get_ideal_block_version(db->network_type(), db->scan_target_height());
+    new_tx.tx.version = cryptonote::transaction::get_max_version_for_hf(hf);
+    new_tx.tx.type = cryptonote::txtype::oxen_name_system;
+    new_tx.fee_per_byte = fee_per_byte;
+    new_tx.fee_per_output = fee_per_output;
+    new_tx.change = change_recipient;
+    new_tx.blink = false;
+
+    std::string reason = "";
+
+    const auto type = ons::parse_ons_type(std::string(type_str));
+    if (!type.has_value())
+      throw std::runtime_error("invalid type provided");
+
+    const auto lower_name = tools::lowercase_ascii_string(name);
+    if (!ons::validate_ons_name(*type, lower_name, &reason))
+      throw std::runtime_error(reason);
+    const auto name_hash = ons::name_to_hash(lower_name);
+
+    ons::mapping_value encrypted_value;
+    if (!ons::mapping_value::validate(nettype, *type, value, &encrypted_value, &reason))
+      throw std::runtime_error(reason);
+
+    if (!encrypted_value.encrypt(lower_name, &name_hash))
+      throw std::runtime_error("Fail to encrypt mapping value="s + value.data());
+
+    ons::generic_owner owner;
+    ons::generic_owner backup_owner;
+
+    if (not owner_str.has_value())
+      owner = ons::make_monero_owner(change_recipient.addr, change_recipient.is_subaddress);
+    else if (not ons::parse_owner_to_generic_owner(nettype, *owner_str, owner, &reason))
+      throw std::runtime_error(reason);
+
+    if (backup_owner_str.has_value() && !ons::parse_owner_to_generic_owner(nettype, *backup_owner_str, backup_owner, &reason))
+      throw std::runtime_error(reason);
+
+    // No prev_txid for initial ons buy
+    crypto::hash prev_txid = {};
+
+    auto ons_buy_data = cryptonote::tx_extra_oxen_name_system::make_buy(
+        owner,
+        backup_owner_str.has_value() ? &backup_owner : nullptr,
+        *type,
+        name_hash,
+        encrypted_value.to_string(),
+        prev_txid);
+
+    new_tx.burn_fixed = ons::burn_needed(cryptonote::get_latest_hard_fork(nettype).version, *type);
+    new_tx.update_change();
+
+    //Finally save the data to the extra field of our transaction
+    cryptonote::add_oxen_name_system_to_tx_extra(new_tx.extra, ons_buy_data);
+    cryptonote::add_burned_amount_to_tx_extra(new_tx.extra, new_tx.burn_fixed);
+
+    select_inputs_and_finalise(new_tx);
+    return new_tx;
+  }
+
+  PendingTransaction
+  TransactionConstructor::create_ons_update_transaction(
+      const std::string& name,
+      const std::string& type_str,
+      std::optional<std::string_view> value,
+      std::optional<std::string_view> owner_str,
+      std::optional<std::string_view> backup_owner_str,
+      const cryptonote::tx_destination_entry& change_recipient,
+      std::shared_ptr<Keyring> keyring
+      )
+  {
+    if (not owner_str.has_value())
+    if (not value.has_value() && not owner_str.has_value() && not backup_owner_str.has_value())
+      throw std::runtime_error("Value, owner and backup owner are not specified. Atleast one field must be specified for updating the ONS record");
+
+    const auto lower_name = tools::lowercase_ascii_string(name);
+    std::string reason;
+    const auto type = ons::parse_ons_type(type_str);
+    if (!type.has_value())
+      throw std::runtime_error("invalid type provided");
+    if (!ons::validate_ons_name(*type, lower_name, &reason))
+      throw std::runtime_error(reason);
+    const auto name_hash = ons::name_to_hash(lower_name);
+
+    auto submit_ons_future = daemon->ons_names_to_owners(oxenc::to_base64(tools::view_guts(name_hash)), ons::db_mapping_type(*type));
+    if (submit_ons_future.wait_for(5s) != std::future_status::ready)
+      throw std::runtime_error("request to daemon for ons_names_to_owners timed out");
+
+    //TODO sean stuff goes here
+    const auto [curr_owner, prev_txid] = submit_ons_future.get();
+
+    ons::mapping_value encrypted_value;
+    if (value.has_value())
+    {
+      if (!ons::mapping_value::validate(nettype, *type, *value, &encrypted_value, &reason))
+        throw std::runtime_error(reason);
+
+      if (!encrypted_value.encrypt(lower_name, &name_hash))
+        throw std::runtime_error("Fail to encrypt name");
+    }
+
+    ons::generic_owner owner;
+    if (owner_str.has_value() && !ons::parse_owner_to_generic_owner(nettype, *owner_str, owner, &reason))
+      throw std::runtime_error(reason);
+
+    ons::generic_owner backup_owner;
+    if (backup_owner_str.has_value() && !ons::parse_owner_to_generic_owner(nettype, *backup_owner_str, backup_owner, &reason))
+      throw std::runtime_error(reason);
+
+    const auto signature = keyring->generate_ons_signature(
+      curr_owner,
+      owner_str.has_value() ? &owner : nullptr,
+      backup_owner_str.has_value() ? &backup_owner : nullptr,
+      encrypted_value,
+      prev_txid,
+      nettype
+      );
+
+    std::vector<cryptonote::tx_destination_entry> recipients;
+    PendingTransaction new_tx(recipients);
+    auto [hf, hf_uint8] = cryptonote::get_ideal_block_version(db->network_type(), db->scan_target_height());
+    new_tx.tx.version = cryptonote::transaction::get_max_version_for_hf(hf);
+    new_tx.tx.type = cryptonote::txtype::oxen_name_system;
+    new_tx.fee_per_byte = fee_per_byte;
+    new_tx.fee_per_output = fee_per_output;
+    new_tx.change = change_recipient;
+    new_tx.blink = false;
+
+    auto ons_update_data = cryptonote::tx_extra_oxen_name_system::make_update(
+        signature,
+        *type,
+        name_hash,
+        encrypted_value.to_string(),
+        owner_str != "" ? &owner : nullptr,
+        backup_owner_str != "" ? &backup_owner : nullptr,
+        prev_txid);
+
+    //Finally save the data to the extra field of our transaction
+    cryptonote::add_oxen_name_system_to_tx_extra(new_tx.extra, ons_update_data);
+    new_tx.update_change();
+
+    select_inputs_and_finalise(new_tx);
+    return new_tx;
+  }
+
 
   // SelectInputs will choose some available unspent outputs from the database and allocate to the
   // transaction can be called multiple times and will add until enough is sufficient

src/wallet3/transaction_constructor.hpp

@@ -5,6 +5,7 @@
 #include <memory>
 #include "pending_transaction.hpp"
 #include "daemon_comms.hpp"
+#include "keyring.hpp"
 #include "decoy_selection/decoy_selection.hpp"
 
 namespace wallet
@@ -33,9 +34,32 @@ namespace wallet
     PendingTransaction
     create_transaction(const std::vector<cryptonote::tx_destination_entry>& recipients, const cryptonote::tx_destination_entry& change_recipient);
 
+    PendingTransaction
+    create_ons_buy_transaction(
+        std::string_view name,
+        std::string_view type_str,
+        std::string_view value,
+        std::optional<std::string_view> owner_str,
+        std::optional<std::string_view> backup_owner_str,
+        const cryptonote::tx_destination_entry& change_recipient
+        );
+
+    PendingTransaction
+    create_ons_update_transaction(
+        const std::string& name,
+        const std::string& type_str,
+        std::optional<std::string_view> value,
+        std::optional<std::string_view> owner_str,
+        std::optional<std::string_view> backup_owner_str,
+        const cryptonote::tx_destination_entry& change_recipient,
+        std::shared_ptr<Keyring> keyring
+        );
+
     uint64_t fee_per_byte = cryptonote::FEE_PER_BYTE_V13;
     uint64_t fee_per_output = cryptonote::FEE_PER_OUTPUT_V18;
 
+    cryptonote::network_type nettype = cryptonote::network_type::TESTNET;
+
     std::unique_ptr<DecoySelector> decoy_selector;
 
    private:

src/wallet3/transaction_scanner.cpp

@@ -10,7 +10,7 @@
 namespace wallet
 {
   namespace log = oxen::log;
-  static auto logcat = log::Cat("wallet.wallet3");
+  static auto logcat = log::Cat("wallet");
 
   std::vector<Output>
   TransactionScanner::scan_received(
@@ -22,7 +22,8 @@ namespace wallet
 
     if (tx_public_keys.empty())
     {
-      log::warning(logcat, "TransactionScanner found no tx public keys in transaction with hash <{}>.", tx.hash);
+      // This sometimes occurs for things like recommission transactions sent by the quorum
+      log::trace(logcat, "TransactionScanner found no tx public keys in transaction with hash <{}>.", tx.hash);
       return {};
     }
     if (tx.tx.vout.size() != tx.global_indices.size())
@@ -31,8 +32,10 @@ namespace wallet
           "Invalid wallet::BlockTX, created outputs count != global indices count.");
     }
 
+    // A derivation is simply the private view key multiplied by the tx public key
+    // do this for every tx public key in the transaction
     auto derivations = wallet_keys->generate_key_derivations(tx_public_keys);
-
+    bool coinbase_transaction = cryptonote::is_coinbase(tx.tx);
     // Output belongs to public key derived as follows:
     //      let `Hs` := hash_to_scalar
     //      let `B`  := recipient public spend key
@@ -43,6 +46,7 @@ namespace wallet
     //      `out_key - Hs(R || output_index) * G == B`
     for (size_t output_index = 0; output_index < tx.tx.vout.size(); output_index++)
     {
+      log::debug(logcat, "scanning output at height: {} output index: {}", height, output_index);
       const auto& output = tx.tx.vout[output_index];
 
       if (auto* output_target = std::get_if<cryptonote::txout_to_key>(&output.target))
@@ -59,17 +63,25 @@ namespace wallet
 
         if (not sub_index)
           continue;  // not ours, move on to the next output
+                     //
+        log::info(logcat, "Found an output belonging to us with subindex: {}:{}", sub_index->major, sub_index->minor);
 
         // TODO: device "conceal derivation" as needed
 
+
         auto key_image = wallet_keys->key_image(
             derivations[derivation_index], output_target->key, output_index, *sub_index);
 
         Output o;
 
-        // TODO: ringct mask returned by reference.  ugh.
-        std::tie(o.amount, o.rct_mask) = wallet_keys->output_amount_and_mask(
-            tx.tx.rct_signatures, derivations[derivation_index], output_index);
+        if (coinbase_transaction)
+        {
+          o.amount = output.amount;
+          o.rct_mask = rct::identity();
+        } else {
+          std::tie(o.amount, o.rct_mask) = wallet_keys->output_amount_and_mask(
+              tx.tx.rct_signatures, derivations[derivation_index], output_index);
+        }
 
         o.key_image = key_image;
         o.subaddress_index = *sub_index;
@@ -115,4 +127,11 @@ namespace wallet
     return spends;
   }
 
+  void
+  TransactionScanner::set_keys(std::shared_ptr<Keyring> keys)
+  {
+    if (wallet_keys != keys)
+      wallet_keys = keys;
+  }
+
 }  // namespace wallet

src/wallet3/transaction_scanner.hpp

@@ -29,6 +29,9 @@ namespace wallet
     std::vector<crypto::key_image>
     scan_spent(const cryptonote::transaction& tx);
 
+    void
+    set_keys(std::shared_ptr<Keyring> keys);
+
    private:
     std::shared_ptr<Keyring> wallet_keys;
     std::shared_ptr<db::Database> db;

src/wallet3/wallet.cpp

@@ -1,6 +1,6 @@
 #include "wallet.hpp"
 
-#include "db_schema.hpp"
+#include "db/walletdb.hpp"
 #include "wallet2½.hpp"
 #include "block.hpp"
 #include "block_tx.hpp"
@@ -12,26 +12,44 @@
 #include <sqlitedb/database.hpp>
 #include <oxenmq/oxenmq.h>
 
-#include "common/fs.h"
 #include <future>
 #include <chrono>
 #include <thread>
 
 #include <iostream>
 
+#include <oxen/log.hpp>
+
+#include <spdlog/sinks/rotating_file_sink.h>
+
 namespace wallet
 {
+  static auto logcat = oxen::log::Cat("wallet");
+
+  fs::path file_path_from_default_datadir(const Config& c, const fs::path& filename)
+  {
+    if (filename.string() == ":memory:")
+        return filename;
+
+    auto file_location = fs::absolute(fs::u8path(c.general.datadir));
+    if (c.general.nettype != "mainnet" && c.general.append_network_type_to_datadir)
+      file_location /= c.general.nettype;
+    file_location /= filename;
+
+    return file_location;
+  }
+
   Wallet::Wallet(
       std::shared_ptr<oxenmq::OxenMQ> omq,
-      std::shared_ptr<Keyring> keys,
+      std::shared_ptr<Keyring> keyring,
       std::shared_ptr<TransactionConstructor> tx_constructor,
       std::shared_ptr<DaemonComms> daemon_comms,
       std::string_view dbFilename,
       std::string_view dbPassword,
       wallet::Config config_in)
       : omq(omq)
-      , db{std::make_shared<WalletDB>(fs::path(dbFilename), dbPassword)}
-      , keys{keys}
+      , db{std::make_shared<WalletDB>(file_path_from_default_datadir(config_in, dbFilename), dbPassword)}
+      , keys{std::move(keyring)}
       , tx_scanner{keys, db}
       , tx_constructor{tx_constructor}
       , daemon_comms{daemon_comms}
@@ -45,9 +63,17 @@ namespace wallet
     if (not tx_constructor)
       this->tx_constructor = std::make_shared<TransactionConstructor>(db, daemon_comms);
 
+    config.omq_rpc.sockname = file_path_from_default_datadir(config, config.omq_rpc.sockname).string();
     omq_server.set_omq(this->omq, config.omq_rpc);
 
     db->create_schema();
+    if (!keys)
+    {
+      const auto db_keys = db->load_keys();
+      keys = std::make_shared<wallet::Keyring>(db_keys->spend_privkey(), db_keys->spend_pubkey(), db_keys->view_privkey(), db_keys->view_pubkey(), nettype);
+      tx_scanner.set_keys(keys);
+    }
+    db->save_keys(keys);
     db->add_address(0, 0, keys->get_main_address());
     last_scan_height = db->last_scan_height();
     scan_target_height = db->scan_target_height();
@@ -56,12 +82,34 @@ namespace wallet
   void
   Wallet::init()
   {
+    keys->expand_subaddresses({config.general.subaddress_lookahead_major, config.general.subaddress_lookahead_minor});
+    oxen::log::reset_level(*oxen::logging::parse_level(config.logging.level));
+    fs::path log_location = "";
+    if (config.logging.save_logs_in_subdirectory)
+      log_location /= config.logging.logdir;
+    log_location /= config.logging.log_filename;
+
+    log_location = file_path_from_default_datadir(config, log_location);
+
+    auto file_sink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(
+        log_location.string(),
+        config.logging.log_file_size_limit,
+        config.logging.extra_files,
+        config.logging.rotate_on_open
+        );
+
+    oxen::log::add_sink(std::move(file_sink));
+    oxen::log::info(logcat, "Writing logs to {}", log_location.string());
+
+    oxen::log::info(logcat, "Remote Daemon set to {}", config.daemon.address);
     request_handler.set_wallet(weak_from_this());
     omq->start();
+    oxen::log::info(logcat, "OMQ started");
     daemon_comms->set_remote(config.daemon.address);
     daemon_comms->register_wallet(*this, last_scan_height + 1 /*next needed block*/,
         true /* update sync height */,
         true /* new wallet */);
+    oxen::log::info(logcat, "Finished wallet init");
   }
 
   Wallet::~Wallet()
@@ -83,7 +131,7 @@ namespace wallet
   uint64_t
   Wallet::get_unlocked_balance()
   {
-    return 0; // TODO: this
+    return db->unlocked_balance();
   }
 
   cryptonote::account_keys
@@ -95,6 +143,7 @@ namespace wallet
   void
   Wallet::add_block(const Block& block)
   {
+    oxen::log::trace(logcat, "add block called with block height {}", block.height);
     auto db_tx = db->db_transaction();
 
     db->store_block(block);
@@ -104,16 +153,19 @@ namespace wallet
       if (auto outputs = tx_scanner.scan_received(tx, block.height, block.timestamp);
           not outputs.empty())
       {
+        oxen::log::info(logcat, "outputs: tx.hash {}, block.height {}, outputs {}", tx.hash, block.height, outputs.size());
         db->store_transaction(tx.hash, block.height, outputs);
       }
 
       if (auto spends = tx_scanner.scan_spent(tx.tx); not spends.empty())
       {
+        oxen::log::info(logcat, "spends: tx.hash {}, block.height {}, spends {}", tx.hash, block.height, spends.size());
         db->store_spends(tx.hash, block.height, spends);
       }
     }
 
     db_tx.commit();
+
     last_scan_height++;
   }
 
@@ -124,11 +176,11 @@ namespace wallet
       return;
 
     if (blocks.size() == 0)
-      //TODO: error handling; this shouldn't be able to happen
-      return;
+      throw std::runtime_error("no blocks sent to add blocks");
 
     if (blocks.front().height > last_scan_height + 1)
     {
+      oxen::log::warning(logcat, "blocks.front height is greater than last scan height, calling register wallet with last scan height of {}", last_scan_height + 1);
       daemon_comms->register_wallet(*this, last_scan_height + 1 /*next needed block*/, true);
       return;
     }

src/wallet3/wallet.hpp

@@ -4,6 +4,7 @@
 #include "transaction_constructor.hpp"
 #include "daemon_comms.hpp"
 #include "keyring.hpp"
+#include "common/fs.h"
 
 #include "config/config.hpp"
 
@@ -21,6 +22,8 @@ namespace oxenmq
 
 namespace wallet
 {
+  fs::path file_path_from_default_datadir(const Config& c, const fs::path& filename);
+
   class WalletDB;
 
   struct Block;
@@ -32,7 +35,7 @@ namespace wallet
    protected:
     Wallet(
         std::shared_ptr<oxenmq::OxenMQ> omq,
-        std::shared_ptr<Keyring> keys,
+        std::shared_ptr<Keyring> keyring,
         std::shared_ptr<TransactionConstructor> tx_constructor,
         std::shared_ptr<DaemonComms> daemon_comms,
         std::string_view dbFilename,
@@ -110,6 +113,7 @@ namespace wallet
     wallet::rpc::OmqServer omq_server;
     bool running = true;
 
+    //TODO get this from config
     cryptonote::network_type nettype = cryptonote::network_type::TESTNET;
   };
 

src/wallet3/walletkeys.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+class WalletKeys {
+public:
+    virtual const crypto::secret_key& spend_privkey() const = 0;  
+    virtual const crypto::public_key& spend_pubkey() const = 0;  
+    virtual const crypto::secret_key& view_privkey() const = 0;  
+    virtual const crypto::public_key& view_pubkey() const = 0;  
+};  
+
+struct DBKeys : public WalletKeys {
+    crypto::secret_key ssk, vsk;
+    crypto::public_key spk, vpk;
+
+    const crypto::secret_key& spend_privkey() const override { return ssk; }
+    const crypto::public_key& spend_pubkey() const override { return spk; }
+    const crypto::secret_key& view_privkey() const override { return vsk; }
+    const crypto::public_key& view_pubkey() const override { return vpk; }
+};

tests/wallet3_tests/db_schema.cpp

@@ -1,6 +1,6 @@
 #include <catch2/catch.hpp>
 
-#include <wallet3/db_schema.hpp>
+#include <wallet3/db/walletdb.hpp>
 #include <sqlitedb/database.hpp>
 
 TEST_CASE("DB Schema", "[wallet,db]")
@@ -17,11 +17,6 @@ TEST_CASE("DB Schema", "[wallet,db]")
 
   REQUIRE(db.db.tableExists("blocks"));
 
-  SECTION("metadata table does not allow row insertion")
-  {
-    REQUIRE_THROWS(db.prepared_exec("INSERT INTO metadata VALUES(1,0,0,0,0);"));
-  }
-
   SECTION("Insert and fetch block")
   {
     REQUIRE_NOTHROW(db.prepared_exec("INSERT INTO blocks VALUES(?,?,?,?);", 42, 0, "Adams", 0));
@@ -75,7 +70,7 @@ TEST_CASE("DB Triggers", "[wallet,db]")
   SECTION("Confirm output insert triggers")
   {
     REQUIRE(db.prepared_get<int64_t>("SELECT amount FROM outputs WHERE id = 0") == 42);
-    REQUIRE(db.prepared_get<int64_t>("SELECT balance FROM metadata WHERE id = 0") == 42);
+    REQUIRE(db.overall_balance() == 42);
   }
 
   REQUIRE_NOTHROW(db.prepared_exec("INSERT INTO blocks VALUES(?,?,?,?);", 1, 0, "bar", 0));
@@ -84,7 +79,7 @@ TEST_CASE("DB Triggers", "[wallet,db]")
 
   SECTION("Confirm spend insert triggers")
   {
-    REQUIRE(db.prepared_get<int64_t>("SELECT balance FROM metadata WHERE id = 0") == 0);
+    REQUIRE(db.overall_balance() == 0);
     REQUIRE(db.prepared_get<int64_t>("SELECT spent_height FROM outputs WHERE key_image = 0") == 1);
   }
 
@@ -101,7 +96,7 @@ TEST_CASE("DB Triggers", "[wallet,db]")
     // balance should be 42, and the spend should be removed.
     // existing output's spend height should be back to 0.
     REQUIRE(db.prepared_get<int>("SELECT COUNT(*) FROM spends;") == 0);
-    REQUIRE(db.prepared_get<int64_t>("SELECT balance FROM metadata WHERE id = 0") == 42);
+    REQUIRE(db.overall_balance() == 42);
     REQUIRE(db.prepared_get<int64_t>("SELECT spent_height FROM outputs WHERE key_image = 0") == 0);
   }
 
@@ -114,7 +109,7 @@ TEST_CASE("DB Triggers", "[wallet,db]")
     // balance should be 0, and the output should be removed.
     // key image should be removed as nothing references it.
     REQUIRE(db.prepared_get<int>("SELECT COUNT(*) FROM outputs;") == 0);
-    REQUIRE(db.prepared_get<int64_t>("SELECT balance FROM metadata WHERE id = 0") == 0);
+    REQUIRE(db.overall_balance() == 0);
     REQUIRE(db.prepared_get<int64_t>("SELECT COUNT(*) FROM key_images;") == 0);
   }
 }

tests/wallet3_tests/jank_wallet.cpp

@@ -46,13 +46,13 @@ int main(int argc, char** argv)
 
   auto keyring = std::make_shared<wallet::Keyring>(spend_priv, spend_pub, view_priv, view_pub, cryptonote::network_type::TESTNET);
 
-  wallet::Config config;
+  wallet::Config config = {};
   auto& comms_config = config.daemon;
   auto& omq_rpc_config = config.omq_rpc;
   auto oxenmq = std::make_shared<oxenmq::OxenMQ>();
   auto comms = std::make_shared<wallet::DefaultDaemonComms>(oxenmq, comms_config);
   config.omq_rpc.sockname = wallet_name + ".sock";
-  auto wallet = wallet::Wallet::create(oxenmq, keyring, nullptr, comms, wallet_name + ".sqlite", "", config);
+  auto wallet = wallet::Wallet::create(oxenmq, keyring, nullptr, comms, ":memory:", "", config);
 
   std::this_thread::sleep_for(1s);
 

tests/wallet3_tests/mock_wallet.hpp

@@ -3,7 +3,7 @@
 #include <wallet3/wallet.hpp>
 #include <wallet3/block.hpp>
 #include <sqlitedb/database.hpp>
-#include <wallet3/db_schema.hpp>
+#include <wallet3/db/walletdb.hpp>
 #include "mock_daemon_comms.hpp"
 
 namespace wallet
@@ -25,14 +25,14 @@ class MockWallet : public Wallet
 {
   public:
 
-    MockWallet() : Wallet({},std::make_shared<Keyring>(),{},std::make_shared<MockDaemonComms>(),":memory:",{}){};
+    MockWallet() : Wallet({},std::make_shared<Keyring>(),{},std::make_shared<MockDaemonComms>(),":memory:","",{}){};
     MockWallet(
         crypto::secret_key _spend_private_key,
         crypto::public_key _spend_public_key,
         crypto::secret_key _view_private_key,
         crypto::public_key _view_public_key,
         cryptonote::network_type _nettype = cryptonote::network_type::TESTNET
-        ) : Wallet({},std::make_shared<Keyring>(_spend_private_key, _spend_public_key, _view_private_key, _view_public_key),{},std::make_shared<MockDaemonComms>(),":memory:",{}){};
+        ) : Wallet({},std::make_shared<Keyring>(_spend_private_key, _spend_public_key, _view_private_key, _view_public_key),{},std::make_shared<MockDaemonComms>(),":memory:","",{}){};
 
     int64_t height = 0;
 

tests/wallet3_tests/tx_creation.cpp

@@ -2,7 +2,7 @@
 #include <catch2/catch.hpp>
 
 #include <wallet3/wallet.hpp>
-#include <wallet3/db_schema.hpp>
+#include <wallet3/db/walletdb.hpp>
 #include <wallet3/transaction_scanner.hpp>
 
 #include <sqlitedb/database.hpp>

utils/parse-tx-extra.py

@@ -0,0 +1,161 @@
+#!/user/bin/python3
+
+## USAGE
+### python3 main.py --tx-extra 0158b9e078dd8c9789be6cc38e7922679734363b87400b9a9bd2ecfd63bffc1a1902090147195636b035b6947a0000114185c0b8bbe6c88966f4393e3069a2b28a979feee27750fc8f527e7cc6ecee00000000000000000000000000000000000000000000000000000000000000000900a1bd51151cc8bc101e523e1a65f27ee83b8e69cbbfefe12320163fd4a22844d6f1595f615164cd19cab8db5c9c00b1a257acf968c6ba72a556cf0b11d3884f480049c2be24ea21c5c91008081ccf401b5a26552b8572bde2d541f8095865c9358184423d738b729f6163301e04c02cccb1fe9680dbc03afe44526856abe99ad2b76d7987b1e333850e1f097900863ba101000000
+
+import argparse
+from enum import IntFlag, auto
+
+parser = argparse.ArgumentParser(description='Decode TX Extra')
+parser.add_argument("--tx-extra", required=True, help="Hex string for the tx txtra to be decoded, type=string")
+args = parser.parse_args()
+
+tx_extra = args.tx_extra
+
+tx_extra_list = list(tx_extra)
+
+tag_dictionary = {
+    "00" : "TX_EXTRA_TAG_PADDING",
+    "01" : "TX_EXTRA_TAG_PUBKEY",
+    "02" : "TX_EXTRA_NONCE",
+    "03" : "TX_EXTRA_MERGE_MINING_TAG",
+    "04" : "TX_EXTRA_TAG_ADDITIONAL_PUBKEYS",
+    "70" : "TX_EXTRA_TAG_SERVICE_NODE_REGISTER",
+    "71" : "TX_EXTRA_TAG_SERVICE_NODE_DEREG_OLD",
+    "72" : "TX_EXTRA_TAG_SERVICE_NODE_WINNER",
+    "73" : "TX_EXTRA_TAG_SERVICE_NODE_CONTRIBUTOR",
+    "74" : "TX_EXTRA_TAG_SERVICE_NODE_PUBKEY",
+    "75" : "TX_EXTRA_TAG_TX_SECRET_KEY",
+    "76" : "TX_EXTRA_TAG_TX_KEY_IMAGE_PROOFS",
+    "77" : "TX_EXTRA_TAG_TX_KEY_IMAGE_UNLOCK",
+    "78" : "TX_EXTRA_TAG_SERVICE_NODE_STATE_CHANGE",
+    "79" : "TX_EXTRA_TAG_BURN",
+    "7A" : "TX_EXTRA_TAG_OXEN_NAME_SYSTEM"
+}
+
+def eat_pubkey_data():
+    global tx_extra_list
+    pubkey = ''.join(tx_extra_list[:64])
+    tx_extra_list = tx_extra_list[64:]
+    return {"pubkey": pubkey}
+
+
+nonce_tag_dictionary = {
+    "00" : "payment_id",
+    "01" : "encrypted_payment_id"
+}
+
+def eat_nonce_data():
+    global tx_extra_list
+    size = (int(''.join(tx_extra_list[:2])) - 1)*2
+    tx_extra_list = tx_extra_list[2:]
+    nonce_tag = ''.join(tx_extra_list[:2])
+    tx_extra_list = tx_extra_list[2:]
+    nonce_data = ''.join(tx_extra_list[:size])
+    tx_extra_list = tx_extra_list[size:]
+    return {nonce_tag_dictionary[nonce_tag]: nonce_data}
+
+ons_type_dictionary = {
+    "00" : "session",
+    "01" : "wallet",
+    "02" : "lokinet",
+    "03" : "lokinet 2 year",
+    "04" : "lokinet 5 year",
+    "05" : "lokinet 10 year"
+}
+
+class ONS_EXTRA_FIELD(IntFlag):
+    OWNER = auto()
+    BACKUP_OWNER = auto()
+    SIGNATURE = auto()
+    ENCRYPTED_VALUE = auto()
+
+class ONS_Extra_Field_Set:
+    def __init__(self, *flags):
+        self._extras = ONS_EXTRA_FIELD(0)  # Initiate no permissions
+        for flag in flags:
+            self._extras |= ONS_EXTRA_FIELD[flag.upper()]
+    def __contains__(self, item):
+        return (self._extras & item) == item
+
+def eat_ons_generic_owner():
+    global tx_extra_list
+    owner_type = ''.join(tx_extra_list[:2])
+    tx_extra_list = tx_extra_list[2:]
+    spend_public_key = ''.join(tx_extra_list[:64])
+    tx_extra_list = tx_extra_list[64:]
+    view_public_key = ''.join(tx_extra_list[:64])
+    tx_extra_list = tx_extra_list[64:]
+    is_subaddress = ''.join(tx_extra_list[:2])
+    tx_extra_list = tx_extra_list[2:]
+    return {"type": owner_type,
+            "spend_public_key": spend_public_key,
+            "view_public_key": view_public_key,
+            "is_subaddress": is_subaddress }
+
+
+def eat_ons_data():
+    global tx_extra_list
+    version = ''.join(tx_extra_list[:2])
+    tx_extra_list = tx_extra_list[2:]
+    ons_type = ''.join(tx_extra_list[:2])
+    tx_extra_list = tx_extra_list[2:]
+    name_hash = ''.join(tx_extra_list[:64])
+    tx_extra_list = tx_extra_list[64:]
+    prev_txid = ''.join(tx_extra_list[:64])
+    tx_extra_list = tx_extra_list[64:]
+    ons_fields = ONS_EXTRA_FIELD(int(''.join(tx_extra_list[:2])))
+    ons_data = {'version': version,
+            'type': ons_type_dictionary[ons_type],
+            'name_hash': name_hash,
+            'prev_txid': prev_txid,
+            'fields': ons_fields}
+    tx_extra_list = tx_extra_list[2:]
+    ons_extra_field_set = ONS_Extra_Field_Set()
+    ons_extra_field_set._extras = ons_fields
+    if ONS_EXTRA_FIELD.OWNER in ons_extra_field_set:
+        ons_data['owner'] = eat_ons_generic_owner()
+    if ONS_EXTRA_FIELD.BACKUP_OWNER in ons_extra_field_set:
+        ons_data['backup_owner'] = eat_ons_generic_owner()
+    if ONS_EXTRA_FIELD.SIGNATURE in ons_extra_field_set:
+        signature = ''.join(tx_extra_list[:64])
+        tx_extra_list = tx_extra_list[64:]
+        ons_data['signature'] = signature
+    if ONS_EXTRA_FIELD.ENCRYPTED_VALUE in ons_extra_field_set:
+        size = (int(''.join(tx_extra_list[:2]), 16))*2
+        tx_extra_list = tx_extra_list[2:]
+        encrypted_value = ''.join(tx_extra_list[:size])
+        tx_extra_list = tx_extra_list[size:]
+        ons_data['encrypted_value'] = encrypted_value
+    return ons_data
+
+def eat_uint64_t():
+    global tx_extra_list
+    amount = ''.join(tx_extra_list[:8*2])
+    tx_extra_list = tx_extra_list[8*2:]
+    return int.from_bytes(bytearray.fromhex(amount), "little", signed=False)
+
+
+def eat_burn():
+    global tx_extra_list
+    return {"amount": eat_uint64_t()}
+
+
+
+eat_data_functions = {
+    "TX_EXTRA_TAG_PUBKEY": eat_pubkey_data,
+    "TX_EXTRA_NONCE": eat_nonce_data,
+    "TX_EXTRA_TAG_OXEN_NAME_SYSTEM": eat_ons_data,
+    "TX_EXTRA_TAG_BURN": eat_burn
+}
+
+# Main loop that reads over every item
+while len(tx_extra_list) > 0:
+    tag = tag_dictionary[''.join(tx_extra_list[:2]).upper()]
+    tx_extra_list = tx_extra_list[2:]
+    print(tag)
+    print(eat_data_functions[tag]())
+
+
+
+