match io loop event order on windows/apple to match linux.

on win32/apple reading packets from the interface does not count as an io operation.
manually trigger pump on win32/apple to pretend that it is an io event.
add platform quark function MaybeWakeUpperLayers on vpn::Interface to manaully wake up the other components on platforms that need that (ones on which packet io is not done via io events).
on non linux platforms, use uv_prepare_t instead of uv_check_t as the former triggers before blocking for io, instead of after. this better matches linux's order of operations in libuv.

cmake/win32.cmake

@@ -15,8 +15,9 @@ if(NOT MSVC_VERSION)
   # to .r[o]data section one after the other!
   add_compile_options(-fno-ident -Wa,-mbig-obj)
   link_libraries( -lws2_32 -lshlwapi -ldbghelp -luser32 -liphlpapi -lpsapi -luserenv)
-  # zmq requires windows xp or higher
+  # the minimum windows version, set to 6 rn because supporting older windows is hell
-  add_definitions(-DWINVER=0x0501 -D_WIN32_WINNT=0x0501)
+  set(_winver 0x0600)
+  add_definitions(-DWINVER=${_winver} -D_WIN32_WINNT=${_winver})
 endif()
 
 if(EMBEDDED_CFG)

llarp/apple/vpn_interface.cpp

@@ -1,12 +1,18 @@
 
 #include "vpn_interface.hpp"
 #include "context.hpp"
+#include <llarp/router/abstractrouter.hpp>
 
 namespace llarp::apple
 {
   VPNInterface::VPNInterface(
-      Context& ctx, packet_write_callback packet_writer, on_readable_callback on_readable)
+      Context& ctx,
-      : m_PacketWriter{std::move(packet_writer)}, m_OnReadable{std::move(on_readable)}
+      packet_write_callback packet_writer,
+      on_readable_callback on_readable,
+      AbstractRouter* router)
+      : m_PacketWriter{std::move(packet_writer)}
+      , m_OnReadable{std::move(on_readable)}
+      , _router{router}
   {
     ctx.loop->call_soon([this] { m_OnReadable(*this); });
   }
@@ -21,6 +27,12 @@ namespace llarp::apple
     return true;
   }
 
+  void
+  VPNInterface::MaybeWakeUpperLayers() const
+  {
+    _router->TriggerPump();
+  }
+
   int
   VPNInterface::PollFD() const
   {

llarp/apple/vpn_interface.hpp

@@ -17,7 +17,10 @@ namespace llarp::apple
     using on_readable_callback = std::function<void(VPNInterface&)>;
 
     explicit VPNInterface(
-        Context& ctx, packet_write_callback packet_writer, on_readable_callback on_readable);
+        Context& ctx,
+        packet_write_callback packet_writer,
+        on_readable_callback on_readable,
+        AbstractRouter* router);
 
     // Method to call when a packet has arrived to deliver the packet to lokinet
     bool
@@ -35,6 +38,9 @@ namespace llarp::apple
     bool
     WritePacket(net::IPPacket pkt) override;
 
+    void
+    MaybeWakeUpperLayers() const override;
+
    private:
     // Function for us to call when we have a packet to emit.  Should return true if the packet was
     // handed off to the OS successfully.
@@ -46,6 +52,8 @@ namespace llarp::apple
     static inline constexpr auto PacketQueueSize = 1024;
 
     thread::Queue<net::IPPacket> m_ReadQueue{PacketQueueSize};
+
+    AbstractRouter* const _router;
   };
 
 }  // namespace llarp::apple

llarp/apple/vpn_platform.cpp

@@ -15,8 +15,9 @@ namespace llarp::apple
       , m_OnReadable{std::move(on_readable)}
   {}
 
-  std::shared_ptr<vpn::NetworkInterface> VPNPlatform::ObtainInterface(vpn::InterfaceInfo)
+  std::shared_ptr<vpn::NetworkInterface>
+  VPNPlatform::ObtainInterface(vpn::InterfaceInfo, AbstractRouter* router)
   {
-    return std::make_shared<VPNInterface>(m_Context, m_PacketWriter, m_OnReadable);
+    return std::make_shared<VPNInterface>(m_Context, m_PacketWriter, m_OnReadable, router);
   }
 }  // namespace llarp::apple

llarp/apple/vpn_platform.hpp

@@ -16,7 +16,8 @@ namespace llarp::apple
         llarp_route_callbacks route_callbacks,
         void* callback_context);
 
-    std::shared_ptr<vpn::NetworkInterface> ObtainInterface(vpn::InterfaceInfo) override;
+    std::shared_ptr<vpn::NetworkInterface>
+    ObtainInterface(vpn::InterfaceInfo, AbstractRouter*) override;
 
     vpn::IRouteManager&
     RouteManager() override

llarp/config/config.cpp

@@ -798,12 +798,12 @@ namespace llarp
       const IpAddress addr{value};
       if (not addr.hasPort())
         throw std::invalid_argument("no port provided in link address");
-      info.interface = addr.toHost();
+      info.m_interface = addr.toHost();
       info.port = *addr.getPort();
     }
     else
     {
-      info.interface = std::string{name};
+      info.m_interface = std::string{name};
 
       std::vector<std::string_view> splits = split(value, ",");
       for (std::string_view str : splits)

llarp/config/config.hpp

@@ -147,7 +147,7 @@ namespace llarp
   {
     struct LinkInfo
     {
-      std::string interface;
+      std::string m_interface;
       int addressFamily = -1;
       uint16_t port = -1;
     };

llarp/ev/ev_libuv.cpp

@@ -241,8 +241,11 @@ namespace llarp::uv
     using event_t = uvw::PollEvent;
     auto handle = m_Impl->resource<uvw::PollHandle>(netif->PollFD());
 #else
-    using event_t = uvw::CheckEvent;
+    // we use a uv_prepare_t because it fires before blocking for new io events unconditionally
-    auto handle = m_Impl->resource<uvw::CheckHandle>();
+    // we want to match what linux does, using a uv_check_t does not suffice as the order of
+    // operations is not what we need.
+    using event_t = uvw::PrepareEvent;
+    auto handle = m_Impl->resource<uvw::PrepareHandle>();
 #endif
     if (!handle)
       return false;
@@ -254,6 +257,10 @@ namespace llarp::uv
         LogDebug("got packet ", pkt.sz);
         if (handler)
           handler(std::move(pkt));
+        // on windows/apple, vpn packet io does not happen as an io action that wakes up the event
+        // loop thus, we must manually wake up the event loop when we get a packet on our interface.
+        // on linux this is a nop
+        netif->MaybeWakeUpperLayers();
       }
     });
 

llarp/ev/vpn.hpp

@@ -9,7 +9,8 @@
 namespace llarp
 {
   struct Context;
-}
+  struct AbstractRouter;
+}  // namespace llarp
 
 namespace llarp::vpn
 {
@@ -59,6 +60,10 @@ namespace llarp::vpn
     /// returns false if we dropped it
     virtual bool
     WritePacket(net::IPPacket pkt) = 0;
+
+    /// idempotently wake up the upper layers as needed (platform dependant)
+    virtual void
+    MaybeWakeUpperLayers() const {};
   };
 
   class IRouteManager
@@ -112,7 +117,7 @@ namespace llarp::vpn
     /// get a new network interface fully configured given the interface info
     /// blocks until ready, throws on error
     virtual std::shared_ptr<NetworkInterface>
-    ObtainInterface(InterfaceInfo info) = 0;
+    ObtainInterface(InterfaceInfo info, AbstractRouter* router) = 0;
 
     /// get owned ip route manager for managing routing table
     virtual IRouteManager&

llarp/handlers/exit.cpp

@@ -443,7 +443,7 @@ namespace llarp
         info.ifname = m_ifname;
         info.addrs.emplace(m_OurRange);
 
-        m_NetIf = GetRouter()->GetVPNPlatform()->ObtainInterface(std::move(info));
+        m_NetIf = GetRouter()->GetVPNPlatform()->ObtainInterface(std::move(info), m_Router);
         if (not m_NetIf)
         {
           llarp::LogError("Could not create interface");

llarp/handlers/tun.cpp

@@ -908,7 +908,7 @@ namespace llarp
 
       try
       {
-        m_NetIf = Router()->GetVPNPlatform()->ObtainInterface(std::move(info));
+        m_NetIf = Router()->GetVPNPlatform()->ObtainInterface(std::move(info), Router());
       }
       catch (std::exception& ex)
       {

llarp/net/sock_addr.hpp

@@ -7,11 +7,6 @@
 #include <winsock2.h>
 #include <ws2tcpip.h>
 #include <wspiapi.h>
-extern "C" const char*
-inet_ntop(int af, const void* src, char* dst, size_t size);
-extern "C" int
-inet_pton(int af, const char* src, void* dst);
-#define inet_aton(x, y) inet_pton(AF_INET, x, y)
 #endif
 
 #include <string_view>

llarp/router/router.cpp

@@ -709,7 +709,7 @@ namespace llarp
           util::memFn(&AbstractRouter::TriggerPump, this),
           util::memFn(&AbstractRouter::QueueWork, this));
 
-      const std::string& key = serverConfig.interface;
+      const std::string& key = serverConfig.m_interface;
       int af = serverConfig.addressFamily;
       uint16_t port = serverConfig.port;
       if (!server->Configure(this, key, af, port))
@@ -1241,10 +1241,6 @@ namespace llarp
 
     LogInfo("have ", _nodedb->NumLoaded(), " routers");
 
-#ifdef _WIN32
-    // windows uses proactor event loop so we need to constantly pump
-    _loop->add_ticker([this] { PumpLL(); });
-#endif
     _loop->call_every(ROUTER_TICK_INTERVAL, weak_from_this(), [this] { Tick(); });
     _running.store(true);
     _startedAt = Now();

llarp/vpn/android.hpp

@@ -92,7 +92,7 @@ namespace llarp::vpn
     {}
 
     std::shared_ptr<NetworkInterface>
-    ObtainInterface(InterfaceInfo info) override
+    ObtainInterface(InterfaceInfo info, AbstractRouter*) override
     {
       return std::make_shared<AndroidInterface>(std::move(info), fd);
     }

llarp/vpn/linux.hpp

@@ -448,7 +448,7 @@ namespace llarp::vpn
 
    public:
     std::shared_ptr<NetworkInterface>
-    ObtainInterface(InterfaceInfo info) override
+    ObtainInterface(InterfaceInfo info, AbstractRouter*) override
     {
       return std::make_shared<LinuxInterface>(std::move(info));
     };

llarp/vpn/win32.hpp

@@ -6,6 +6,7 @@
 #include <fcntl.h>
 #include <llarp/util/thread/queue.hpp>
 #include <llarp/ev/vpn.hpp>
+#include <llarp/router/abstractrouter.hpp>
 
 // DDK macros
 #define CTL_CODE(DeviceType, Function, Method, Access) \
@@ -177,6 +178,8 @@ namespace llarp::vpn
 
     InterfaceInfo m_Info;
 
+    AbstractRouter* const _router;
+
     static std::wstring
     get_win_sys_path()
     {
@@ -220,7 +223,8 @@ namespace llarp::vpn
       return converter.to_bytes(wcmd);
     }
 
-    Win32Interface(InterfaceInfo info) : m_ReadQueue{1024}, m_Info{std::move(info)}
+    Win32Interface(InterfaceInfo info, AbstractRouter* router)
+        : m_ReadQueue{1024}, m_Info{std::move(info)}, _router{router}
     {
       DWORD len;
 
@@ -401,6 +405,12 @@ namespace llarp::vpn
         thread.join();
     }
 
+    virtual void
+    MaybeWakeUpperLayers() const override
+    {
+      _router->TriggerPump();
+    }
+
     int
     PollFD() const override
     {
@@ -541,8 +551,8 @@ namespace llarp::vpn
       Execute(RouteCommand() + " " + cmd + " c000::/2 " + ipv6.ToString());
 
       ifname.back()++;
-      Execute(RouteCommand() + " " + cmd + " 0.0.0.0 MASK 128.0.0.0 " + ifname);
+      Execute(RouteCommand() + " " + cmd + " 0.0.0.0 MASK 128.0.0.0 " + ifname + " METRIC 2");
-      Execute(RouteCommand() + " " + cmd + " 128.0.0.0 MASK 128.0.0.0 " + ifname);
+      Execute(RouteCommand() + " " + cmd + " 128.0.0.0 MASK 128.0.0.0 " + ifname + " METRIC 2");
     }
 
     void
@@ -629,12 +639,13 @@ namespace llarp::vpn
 
    public:
     std::shared_ptr<NetworkInterface>
-    ObtainInterface(InterfaceInfo info) override
+    ObtainInterface(InterfaceInfo info, AbstractRouter* router) override
     {
-      auto netif = std::make_shared<Win32Interface>(std::move(info));
+      auto netif = std::make_shared<Win32Interface>(std::move(info), router);
       netif->Start();
       return netif;
     };
+
     IRouteManager&
     RouteManager() override
     {

pybind/llarp/config.cpp

@@ -77,7 +77,7 @@ namespace llarp
             "setOutboundLink",
             [](LinksConfig& self, std::string interface, int family, uint16_t port) {
               LinksConfig::LinkInfo info;
-              info.interface = std::move(interface);
+              info.m_interface = std::move(interface);
               info.addressFamily = family;
               info.port = port;
               self.m_OutboundLink = std::move(info);
@@ -86,7 +86,7 @@ namespace llarp
             "addInboundLink",
             [](LinksConfig& self, std::string interface, int family, uint16_t port) {
               LinksConfig::LinkInfo info;
-              info.interface = std::move(interface);
+              info.m_interface = std::move(interface);
               info.addressFamily = family;
               info.port = port;
               self.m_InboundLinks.push_back(info);