kiiiiiick

- nodes now send their RC to the bootstrap seed on making a request
- allows the bootstrap seed to distribute RCs

external/oxen-libquic

@@ -1 +1 @@
-Subproject commit c9271e87eaa2d6bcbdf4e49936fcce2b7bc9e693
+Subproject commit ad8904debc55f9495667e8571ee5bb963156d54e

llarp/link/connection.cpp

@@ -3,7 +3,7 @@
 namespace llarp::link
 {
   Connection::Connection(
-      std::shared_ptr<oxen::quic::connection_interface>& c,
+      const std::shared_ptr<oxen::quic::connection_interface>& c,
       std::shared_ptr<oxen::quic::BTRequestStream>& s,
       const RemoteRC& rc)
       : conn{c}, control_stream{s}, remote_rc{std::move(rc)}

llarp/link/connection.hpp

@@ -18,7 +18,7 @@ namespace llarp::link
     bool remote_is_relay{true};
 
     Connection(
-        std::shared_ptr<oxen::quic::connection_interface>& c,
+        const std::shared_ptr<oxen::quic::connection_interface>& c,
         std::shared_ptr<oxen::quic::BTRequestStream>& s,
         const RemoteRC& rc);
   };

llarp/link/link_manager.cpp

@@ -135,6 +135,11 @@ namespace llarp
           [this, msg = std::move(m)]() mutable { handle_gossip_rc(std::move(msg)); });
     });
 
+    s->register_command("bfetch_rcs"s, [this, rid](oxen::quic::message m) {
+      _router.loop()->call(
+          [this, msg = std::move(m)]() mutable { handle_fetch_bootstrap_rcs(std::move(msg)); });
+    });
+
     for (auto& method : direct_requests)
     {
       s->register_command(
@@ -172,15 +177,26 @@ namespace llarp
     tls_creds->set_key_verify_callback([this](const ustring_view& key, const ustring_view&) {
       bool result = false;
       RouterID other{key.data()};
+
       if (auto itr = rids_pending_verification.find(other); itr != rids_pending_verification.end())
       {
+        verified_rids[other] = itr->second;
         rids_pending_verification.erase(itr);
         result = true;
       }
+
       if (_router.node_db()->has_rc(other))
         result = true;
 
-      log::critical(logcat, "{}uccessfully verified connection to {}!", result ? "S" : "Un", other);
+      // TODO: discuss pubkey verification for bootstraps connecting to seed node
+      if (_router.is_bootstrap_seed())
+      {
+        log::warning(logcat, "Allowing connection -- we are bootstrap seed");
+        result = true;
+      }
+
+      log::critical(
+          logcat, "{}uccessfully verified connection to {}!", result ? "S" : "Uns", other);
       return result;
     });
     if (_router.is_service_node())
@@ -203,7 +219,8 @@ namespace llarp
               register_commands(s);
               return s;
             }
-            return nullptr;
+
+            return e.make_shared<oxen::quic::Stream>(c, e);
           });
     }
     return ep;
@@ -341,7 +358,7 @@ namespace llarp
     const auto& remote_addr = rc.addr();
     const auto& rid = rc.router_id();
 
-    rids_pending_verification.insert(rid);
+    rids_pending_verification[rid] = rc;
 
     // TODO: confirm remote end is using the expected pubkey (RouterID).
     // TODO: ALPN for "client" vs "relay" (could just be set on endpoint creation)
@@ -358,13 +375,39 @@ namespace llarp
     log::warning(quic_cat, "Failed to begin establishing connection to {}", remote_addr);
   }
 
+  void
+  LinkManager::on_inbound_conn(oxen::quic::connection_interface& ci)
+  {
+    const auto& scid = ci.scid();
+    RouterID rid{ci.remote_key()};
+
+    const auto& rc = verified_rids[rid];
+    ep.connid_map.emplace(scid, rid);
+    auto [itr, b] = ep.conns.emplace(rid, nullptr);
+
+    auto control_stream = ci.queue_stream<oxen::quic::BTRequestStream>([](oxen::quic::Stream& s,
+                                                                          uint64_t error_code) {
+      log::warning(
+          logcat, "BTRequestStream closed unexpectedly (ec:{}); closing connection...", error_code);
+      s.conn.close_connection(error_code);
+    });
+    itr->second = std::make_shared<link::Connection>(ci.shared_from_this(), control_stream, rc);
+    log::critical(logcat, "Successfully configured inbound connection fom {}; storing RC...", rid);
+  }
+
   // TODO: should we add routes here now that Router::SessionOpen is gone?
   void
   LinkManager::on_conn_open(oxen::quic::connection_interface& ci)
   {
     _router.loop()->call([this, &conn_interface = ci]() {
-      const auto& scid = conn_interface.scid();
-      const auto& rid = ep.connid_map[scid];
+      const auto rid = RouterID{conn_interface.remote_key()};
+      const auto& remote = conn_interface.remote();
+
+      if (conn_interface.is_inbound())
+      {
+        log::critical(logcat, "Inbound connection fom {} (remote:{})", rid, remote);
+        on_inbound_conn(conn_interface);
+      }
 
       log::critical(
           logcat,
@@ -376,6 +419,7 @@ namespace llarp
       // messages to the remote
       if (auto itr = pending_conn_msg_queue.find(rid); itr != pending_conn_msg_queue.end())
       {
+        log::critical(logcat, "Clearing pending queue for RID:{}", rid);
         auto& que = itr->second;
 
         while (not que.empty())
@@ -385,6 +429,7 @@ namespace llarp
           if (m.is_control)
           {
             auto& msg = reinterpret_cast<PendingControlMessage&>(m);
+            log::critical(logcat, "Dispatching {} request!", msg.endpoint);
             ep.conns[rid]->control_stream->command(msg.endpoint, msg.body, msg.func);
           }
           else
@@ -395,7 +440,9 @@ namespace llarp
 
           que.pop_front();
         }
+        return;
       }
+      log::warning(logcat, "No pending queue to clear for RID:{}", rid);
     });
   };
 
@@ -405,7 +452,7 @@ namespace llarp
     _router.loop()->call([this, &conn_interface = ci, error_code = ec]() {
       const auto& scid = conn_interface.scid();
 
-      log::debug(quic_cat, "Purging quic connection CID:{} (ec: {})", scid, error_code);
+      log::critical(quic_cat, "Purging quic connection CID:{} (ec: {})", scid, error_code);
 
       if (const auto& c_itr = ep.connid_map.find(scid); c_itr != ep.connid_map.end())
       {
@@ -423,7 +470,7 @@ namespace llarp
 
         ep.connid_map.erase(c_itr);
 
-        log::debug(quic_cat, "Quic connection CID:{} purged successfully", scid);
+        log::critical(quic_cat, "Quic connection CID:{} purged successfully", scid);
       }
     });
   }
@@ -573,6 +620,15 @@ namespace llarp
       const RemoteRC& source, std::string payload, std::function<void(oxen::quic::message m)> func)
   {
     _router.loop()->call([this, source, payload, f = std::move(func)]() {
+
+      if (auto conn = ep.get_conn(source); conn)
+      {
+        log::critical(logcat, "Dispatched bootstrap fetch request!");
+        conn->control_stream->command("bfetch_rcs"s, std::move(payload), std::move(f));
+        return;
+      }
+      
+      log::critical(logcat, "Queuing bootstrap fetch request");
       auto pending = PendingControlMessage(std::move(payload), "bfetch_rcs"s, f);
 
       auto [itr, b] = pending_conn_msg_queue.emplace(source.router_id(), MessageQueue());
@@ -589,11 +645,14 @@ namespace llarp
     assert(_router.is_service_node());
 
     const auto& rcs = node_db->get_rcs();
+    RemoteRC remote;
     size_t quantity;
 
     try
     {
       oxenc::bt_dict_consumer btdc{m.body()};
+      btdc.required("local");
+      remote = RemoteRC{btdc.consume_dict_consumer()};
       quantity = btdc.require<size_t>("quantity");
     }
     catch (const std::exception& e)
@@ -603,6 +662,8 @@ namespace llarp
       return;
     }
 
+    node_db->put_rc(remote);
+
     auto rc_size = rcs.size();
     auto now = llarp::time_now_ms();
     size_t i = 0;

llarp/link/link_manager.hpp

@@ -180,7 +180,10 @@ namespace llarp
     std::unordered_map<RouterID, MessageQueue> pending_conn_msg_queue;
     // when establishing a connection, the rid of the remote is placed here to be cross-
     // checked by the tls verification callback
-    std::set<RouterID> rids_pending_verification;
+    std::map<RouterID, RemoteRC> rids_pending_verification;
+    // in the interim of verifying an inbound connection and the creation of its link::Connection
+    // object, we store the rid and rc here
+    std::map<RouterID, RemoteRC> verified_rids;
 
     util::DecayingHashSet<RouterID> clients{path::DEFAULT_LIFETIME};
 
@@ -203,6 +206,9 @@ namespace llarp
     void
     recv_control_message(oxen::quic::message msg);
 
+    void
+    on_inbound_conn(oxen::quic::connection_interface& ci);
+
     void
     on_conn_open(oxen::quic::connection_interface& ci);
 
@@ -414,6 +420,8 @@ namespace llarp
         connid_map.emplace(conn_interface->scid(), rc.router_id());
         auto [itr, b] = conns.emplace(rc.router_id(), nullptr);
 
+        log::critical(logcat, "Establishing connection to {}...", rc.router_id());
+
         auto control_stream = conn_interface->template get_new_stream<oxen::quic::BTRequestStream>(
             [](oxen::quic::Stream& s, uint64_t error_code) {
               log::warning(
@@ -422,6 +430,7 @@ namespace llarp
                   error_code);
               s.conn.close_connection(error_code);
             });
+
         itr->second = std::make_shared<link::Connection>(conn_interface, control_stream, rc);
 
         return true;

llarp/messages/fetch.hpp

@@ -37,10 +37,12 @@ namespace llarp
 
   namespace BootstrapFetchMessage
   {
+    // the LocalRC is converted to a RemoteRC type to send to the bootstrap seed
     inline static std::string
-    serialize(size_t quantity)
+    serialize(const RemoteRC& local_rc, size_t quantity)
     {
       oxenc::bt_dict_producer btdp;
+      btdp.append_encoded("local", local_rc.view());
       btdp.append("quantity", quantity);
       return std::move(btdp).str();
     }

llarp/nodedb.cpp

@@ -670,7 +670,8 @@ namespace llarp
 
     _router.link_manager().fetch_bootstrap_rcs(
         _bootstraps->current(),
-        BootstrapFetchMessage::serialize(BOOTSTRAP_SOURCE_COUNT),
+        BootstrapFetchMessage::serialize(
+            _router.router_contact.to_remote(), BOOTSTRAP_SOURCE_COUNT),
         [this](oxen::quic::message m) mutable {
           if (not m)
           {

llarp/router/router.cpp

@@ -666,7 +666,12 @@ namespace llarp
 
     clear_bad_rcs();
 
-    if (_bootstrap_rc_list->empty() and not conf.bootstrap.seednode)
+    _bootstrap_seed = conf.bootstrap.seednode;
+
+    if (_bootstrap_seed)
+      log::critical(logcat, "We are a bootstrap seed node!");
+
+    if (_bootstrap_rc_list->empty() and not _bootstrap_seed)
     {
       log::warning(logcat, "Warning: bootstrap list is empty and we are not a seed node");
 
@@ -698,9 +703,6 @@ namespace llarp
 
     node_db()->set_bootstrap_routers(std::move(_bootstrap_rc_list));
 
-    if (conf.bootstrap.seednode)
-      log::critical(logcat, "We are a bootstrap seed node!");
-
     // TODO: RC refactor here
     if (_is_service_node)
       init_inbounds();
@@ -765,7 +767,8 @@ namespace llarp
   Router::report_stats()
   {
     const auto now = llarp::time_now_ms();
-    log::info(
+
+    log::critical(
         logcat,
         "{} RCs loaded with {} bootstrap peers and {} router connections!",
         _node_db->num_rcs(),
@@ -774,15 +777,17 @@ namespace llarp
 
     if (is_service_node())
     {
-      log::info(
+      log::critical(
           logcat,
           "Local service node has {} client connections since last RC update ({} to expiry)",
           num_client_connections(),
           router_contact.age(now),
           router_contact.time_to_expiry(now));
     }
+
     if (_last_stats_report > 0s)
       log::info(logcat, "Last reported stats time {}", now - _last_stats_report);
+
     _last_stats_report = now;
   }
 

llarp/router/router.hpp

@@ -88,6 +88,7 @@ namespace llarp
 
     bool _testnet = false;
     bool _testing_disabled = false;
+    bool _bootstrap_seed = false;
 
     consensus::reachability_testing router_testing;
 
@@ -157,6 +158,12 @@ namespace llarp
     std::chrono::system_clock::time_point next_bootstrap_attempt{last_rc_gossip};
 
    public:
+    bool
+    is_bootstrap_seed() const
+    {
+      return _bootstrap_seed;
+    }
+
     int
     required_num_client_conns() const
     {

llarp/router_contact.hpp

@@ -207,6 +207,8 @@ namespace llarp
     bt_load(oxenc::bt_dict_consumer& data);
   };
 
+  struct RemoteRC;
+
   /// Extension of RouterContact used to store a local "RC," and inserts a RouterContact by
   /// re-parsing and sending it out. This sub-class contains a pubkey and all the other attributes
   /// required for signing and serialization
@@ -236,6 +238,9 @@ namespace llarp
     explicit LocalRC(std::string payload, const SecretKey sk);
     ~LocalRC() = default;
 
+    RemoteRC
+    to_remote();
+
     void
     resign();
 

llarp/router_contact_local.cpp

@@ -27,6 +27,12 @@ namespace llarp
     resign();
   }
 
+  RemoteRC
+  LocalRC::to_remote()
+  {
+    return RemoteRC{view()};
+  }
+
   LocalRC::LocalRC(std::string payload, const SecretKey sk) : _secret_key{std::move(sk)}
   {
     _router_id = llarp::seckey_to_pubkey(_secret_key);

llarp/router_id.hpp

@@ -21,6 +21,9 @@ namespace llarp
     RouterID(const Data& data) : PubKey(data)
     {}
 
+    RouterID(ustring_view data) : PubKey(data.data())
+    {}
+
     util::StatusObject
     ExtractStatus() const;