2018-01-07 06:05:16 +01:00
|
|
|
# Copyright (c) 2014-2018, The Monero Project
|
2014-09-11 08:25:07 +02:00
|
|
|
#
|
|
|
|
|
# All rights reserved.
|
|
|
|
|
#
|
|
|
|
|
# Redistribution and use in source and binary forms, with or without modification, are
|
|
|
|
|
# permitted provided that the following conditions are met:
|
|
|
|
|
#
|
|
|
|
|
# 1. Redistributions of source code must retain the above copyright notice, this list of
|
|
|
|
|
# conditions and the following disclaimer.
|
|
|
|
|
#
|
|
|
|
|
# 2. Redistributions in binary form must reproduce the above copyright notice, this list
|
|
|
|
|
# of conditions and the following disclaimer in the documentation and/or other
|
|
|
|
|
# materials provided with the distribution.
|
|
|
|
|
#
|
|
|
|
|
# 3. Neither the name of the copyright holder nor the names of its contributors may be
|
|
|
|
|
# used to endorse or promote products derived from this software without specific
|
|
|
|
|
# prior written permission.
|
|
|
|
|
#
|
|
|
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
|
|
|
|
|
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
|
|
|
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
|
|
|
|
|
# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
|
|
|
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
|
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
|
|
|
|
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
|
|
|
|
|
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
#
|
|
|
|
|
# Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
|
|
|
|
|
|
2014-09-11 08:22:49 +02:00
|
|
|
|
2014-10-06 15:00:06 +02:00
|
|
|
# We always compile if we are building statically to reduce static dependency issues...
|
|
|
|
|
# ...except for FreeBSD, because FreeBSD is a special case that doesn't play well with
|
|
|
|
|
# others.
|
2014-09-24 21:23:23 +02:00
|
|
|
|
Static builds: make usable binaries from cmake
This adds a static dependency script for libraries like boost, unbound,
etc. to cmake, invokable with:
cmake .. -DBUILD_STATIC_DEPS=ON
which downloads and builds static versions of all our required
dependencies (boost, unbound, openssl, ncurses, etc.). It also implies
-DSTATIC=ON to build other vendored deps (like miniupnpc, lokimq) as
static as well.
Unlike the contrib/depends system, this is easier to maintain (one
script using nicer cmake with functions instead of raw Makefile
spaghetti code), and isn't concerned with reproducible builds -- this
doesn't rebuild the compiler, for instance. It also works with the
existing build system so that it is simply another way to invoke the
cmake build scripts but doesn't require any external tooling.
This works on Linux, Mac, and Windows.
Some random comments on this commit (for preserving history):
- Don't use target_link_libraries on imported targets. Newer cmake is
fine with it, but Bionic's cmake doesn't like it but seems okay with
setting the properties directly.
- This rebuilds libzmq and libsodium, even though there is some
provision already within loki-core to do so: however, the existing
embedded libzmq fails with the static deps because it uses libzmq's
cmake build script, which relies on pkg-config to find libsodium which
ends up finding the system one (or not finding any), rather than the one
we build with DownloadLibSodium. Since both libsodium and libzmq are
faily simple builds it seemed easiest to just add them to the cmake
static build rather than trying to shoehorn the current code into the
static build script.
- Half of the protobuf build system ignores CC/CXX just because Google,
and there's no documentation anywhere except for a random closed bug
report about needing to set these other variables (CC_FOR_BUILD,
CXX_FOR_BUILD) instead, but you need to. Thanks Google.
- The boost build is set to output very little because even the minimum
-d1 output level spams ~15k lines of output just for the headers it
installs.
2020-06-11 20:19:44 +02:00
|
|
|
if(NOT STATIC AND NOT BUILD_STATIC_DEPS)
|
2020-06-12 07:45:25 +02:00
|
|
|
find_package(PkgConfig REQUIRED)
|
2018-09-02 16:51:23 +02:00
|
|
|
endif()
|
2014-10-06 15:00:06 +02:00
|
|
|
|
2022-05-30 20:23:06 +02:00
|
|
|
macro(system_or_submodule BIGNAME smallname pkgconf subdir)
|
|
|
|
|
option(FORCE_${BIGNAME}_SUBMODULE "force using ${smallname} submodule" OFF)
|
|
|
|
|
if(NOT STATIC AND NOT FORCE_${BIGNAME}_SUBMODULE)
|
|
|
|
|
pkg_check_modules(${BIGNAME} ${pkgconf} IMPORTED_TARGET)
|
|
|
|
|
endif()
|
|
|
|
|
if(${BIGNAME}_FOUND)
|
|
|
|
|
add_library(${smallname} INTERFACE)
|
|
|
|
|
if(NOT TARGET PkgConfig::${BIGNAME} AND CMAKE_VERSION VERSION_LESS "3.21")
|
|
|
|
|
# Work around cmake bug 22180 (PkgConfig::THING not set if no flags needed)
|
|
|
|
|
else()
|
|
|
|
|
target_link_libraries(${smallname} INTERFACE PkgConfig::${BIGNAME})
|
|
|
|
|
endif()
|
|
|
|
|
message(STATUS "Found system ${smallname} ${${BIGNAME}_VERSION}")
|
cmake modernization
The archaic (i.e. decade old) cmake usage here really got in the way of
trying to properly use newer libraries (like lokimq), so this undertakes
overhauling it considerably to make it much more sane (and significantly
reduce the size).
I left more of the architecture-specific bits in the top-level
CMakeLists.txt intact; most of the efforts here are about properly
loading dependencies, specifying dependencies and avoiding a whole pile
of cmake antipatterns.
This bumps the required cmake version to 3.5, which is what xenial comes
with.
- extensive use of interface libraries to include libraries,
definitions, and include paths
- use Boost::whatever instead of ${Boost_WHATEVER_LIBRARY}. The
interface targets are (again) much better as they also give you any
needed include or linking flags without needing to worry about them.
- don't list header files when building things. This has *never* been
correct cmake usage (cmake has always known how to wallet_rpc_headers
the headers that .cpp files include to know about build changes).
- remove the loki_add_library monstrosity; it breaks target names and
makes compiling less efficient because the author couldn't figure out
how to link things together.
- make loki_add_executable take the output filename, and set the output
path to bin/ and install to bin because *every single usage* of
loki_add_executable was immediately followed by setting the output
filename and setting the output path to bin/ and installing to bin.
- move a bunch of crap that is only used in one particular
src/whatever/CMakeLists.txt into that particular CMakeLists.txt instead
of the top level CMakeLists.txt (or src/CMakeLists.txt).
- Remove a bunch of redundant dependencies; most of them look like they
were just copy-and-pasted in, and many more aren't needed (since they
are implied by the PUBLIC linking of other dependencies).
- Removed `die` since it just does a FATAL_ERROR, but adds color (which
is useless since CMake already makes FATAL_ERRORs perfectly visible).
- Change the way LOKI_DAEMON_AND_WALLET_ONLY works to just change the
make targets to daemon and simplewallet rather than changing the build
process (this should make it faster, too, since there are various other
things that will be excluded).
2020-03-03 04:57:08 +01:00
|
|
|
else()
|
2022-05-30 20:23:06 +02:00
|
|
|
message(STATUS "using ${smallname} submodule")
|
|
|
|
|
add_subdirectory(${subdir})
|
2019-12-01 06:24:05 +01:00
|
|
|
endif()
|
2022-05-30 20:23:06 +02:00
|
|
|
if(NOT TARGET ${smallname}::${smallname})
|
|
|
|
|
add_library(${smallname}::${smallname} ALIAS ${smallname})
|
2019-12-01 06:24:05 +01:00
|
|
|
endif()
|
2022-05-30 20:23:06 +02:00
|
|
|
endmacro()
|
2019-12-01 06:24:05 +01:00
|
|
|
|
2022-05-30 20:23:06 +02:00
|
|
|
system_or_submodule(OXENC oxenc liboxenc>=1.0.3 oxen-encoding)
|
2022-09-22 21:01:16 +02:00
|
|
|
system_or_submodule(OXENMQ oxenmq liboxenmq>=1.2.13 oxen-mq)
|
2014-10-06 15:00:06 +02:00
|
|
|
|
2015-03-17 23:55:24 +01:00
|
|
|
add_subdirectory(db_drivers)
|
2019-07-01 06:19:04 +02:00
|
|
|
add_subdirectory(randomx EXCLUDE_FROM_ALL)
|
2021-08-13 00:48:02 +02:00
|
|
|
add_subdirectory(date EXCLUDE_FROM_ALL)
|
Replace epee http rpc server with uWebSockets
This replaces the NIH epee http server which does not work all that well
with an external C++ library called uWebSockets. Fundamentally this
gives the following advantages:
- Much less code to maintain
- Just one thread for handling HTTP connections versus epee's pool of
threads
- Uses existing LokiMQ job server and existing thread pool for handling
the actual tasks; they are processed/scheduled in the same "rpc" or
"admin" queues as lokimq rpc calls. One notable benefit is that "admin"
rpc commands get their own queue (and thus cannot be delayed by long rpc
commands). Currently the lokimq threads and the http rpc thread pool
and the p2p thread pool and the job queue thread pool and the dns lookup
thread pool and... are *all* different thread pools; this is a step
towards consolidating them.
- Very little mutex contention (which has been a major problem with epee
RPC in the past): there is one mutex (inside uWebSockets) for putting
responses back into the thread managing the connection; everything
internally gets handled through (lock-free) lokimq inproc sockets.
- Faster RPC performance on average, and much better worst case
performance. Epee's http interface seems to have some race condition
that ocassionally stalls a request (even a very simple one) for a dozen
or more seconds for no good reason.
- Long polling gets redone here to no longer need threads; instead we
just store the request and respond when the thread pool, or else in a
timer (that runs once/second) for timing out long polls.
---
The basic idea of how this works from a high level:
We launch a single thread to handle HTTP RPC requests and response data.
This uWebSockets thread is essentially running an event loop: it never
actually handles any logic; it only serves to shuttle data that arrives
in a request to some other thread, and then, at some later point, to
send some reply back to that waiting connection. Everything is
asynchronous and non-blocking here: the basic uWebSockets event loop
just operates as things arrive, passes it off immediately, and goes back
to waiting for the next thing to arrive.
The basic flow is like this:
0. uWS thread -- listens on localhost:22023
1. uWS thread -- incoming request on localhost:22023
2. uWS thread -- fires callback, which injects the task into the LokiMQ job queue
3. LMQ main loop -- schedules it as an RPC job
4. LMQ rpc thread -- Some LokiMQ thread runs it, gets the result
5. LMQ rpc thread -- Result gets queued up for the uWS thread
6. uWS thread -- takes the request and starts sending it
(asynchronously) back to the requestor.
In more detail:
uWebSockets has registered has registered handlers for non-jsonrpc
requests (legacy JSON or binary). If the port is restricted then admin
commands get mapped to a "Access denied" response handler, otherwise
public commands (and admin commands on an unrestricted port) go to the
rpc command handler.
POST requests to /json_rpc have their own handler; this is a little
different than the above because it has to parse the request before it
can determine whether it is allowed or not, but once this is done it
continues roughly the same as legacy/binary requests.
uWebSockets then listens on the given IP/port for new incoming requests,
and starts listening for requests in a thread (we own this thread).
When a request arrives, it fires the event handler for that request.
(This may happen multiple times, if the client is sending a bunch of
data in a POST request). Once we have the full request, we then queue
the job in LokiMQ, putting it in the "rpc" or "admin" command
categories. (The one practical different here is that "admin" is
configured to be allowed to start up its own thread if all other threads
are busy, while "rpc" commands are prioritized along with everything
else.) LokiMQ then schedules this, along with native LokiMQ "rpc." or
"admin." requests.
When a LMQ worker thread becomes available, the RPC command gets called
in it and runs. Whatever output it produces (or error message, if it
throws) then gets wrapped up in jsonrpc boilerplate (if necessary), and
delivered to the uWebSockets thread to be sent in reply to that request.
uWebSockets picks up the data and sends whatever it can without
blocking, then buffers whatever it couldn't send to be sent again in a
later event loop iteration once the requestor can accept more data.
(This part is outside lokid; we only have to give uWS the data and let
it worry about delivery).
---
PR specifics:
Things removed from this PR:
1. ssl settings; with this PR the HTTP RPC interface is plain-text. The
previous default generated a self-signed certificate for the server on
startup and then the client accepted any certificate. This is actually
*worse* than unencrypted because it is entirely MITM-readable and yet
might make people think that their RPC communication is encrypted, and
setting up actual certificates is difficult enough that I think most
people don't bother.
uWebSockets *does* support HTTPS, and we could glue the existing options
into it, but I'm not convinced it's worthwhile: it works much better to
put HTTPS in a front-end proxy holding the certificate that proxies
requests to the backend (which can then listen in restricted mode on
some localhost port). One reason this is better is that it is much
easier to reload and/or restart such a front-end server, while
certificate updates with lokid require a full restart. Another reason
is that you get an error page instead of a timeout if something is wrong
with the backend. Finally we also save having to generate a temporary
certificate on *every* lokid invocation.
2. HTTP Digest authentication. Digest authentication is obsolete (and
was already obsolete when it got added to Monero). HTTP-Digest was
originally an attempt to provide a password authentication mechanism
that does not leak the password in transit, but still required that the
server know the password. It only has marginal value against replay
attacks, and is made entirely obsolete by sending traffic over HTTPS
instead. No client out there supports Digest but *not* Basic auth, and
so given the limited usefulness it seems pointless to support more than
Basic auth for HTTP RPC login.
What's worse is that epee's HTTP Digest authentication is a terrible
implementation: it uses boost::spirit -- a recursive descent parser
meant for building complex language grammars -- just to parse a single
HTTP header for Digest auth. This is a big load of crap that should
never have been accepted upstream, and that we should get rid of (even
if we wanted to support Digest auth it takes less than 100 lines of code
to do it when *not* using a recursive descent parser).
2020-06-29 01:23:06 +02:00
|
|
|
|
2021-08-13 01:12:14 +02:00
|
|
|
set(JSON_BuildTests OFF CACHE INTERNAL "")
|
2021-08-12 18:33:05 +02:00
|
|
|
set(JSON_MultipleHeaders ON CACHE BOOL "") # Allows multi-header nlohmann use
|
2021-08-13 01:12:14 +02:00
|
|
|
add_subdirectory(nlohmann-json EXCLUDE_FROM_ALL)
|
2022-09-27 05:11:54 +02:00
|
|
|
if(BUILD_STATIC_DEPS)
|
|
|
|
|
set(OXEN_LOGGING_FORCE_SUBMODULES TRUE CACHE BOOL "" FORCE)
|
|
|
|
|
endif()
|
2022-12-16 18:49:17 +01:00
|
|
|
set(OXEN_LOGGING_SOURCE_ROOT "${PROJECT_SOURCE_DIR}/src;${PROJECT_SOURCE_DIR}" CACHE INTERNAL "")
|
2022-09-13 08:57:56 +02:00
|
|
|
add_subdirectory(oxen-logging)
|
2021-08-12 18:33:05 +02:00
|
|
|
|
Replace epee http rpc server with uWebSockets
This replaces the NIH epee http server which does not work all that well
with an external C++ library called uWebSockets. Fundamentally this
gives the following advantages:
- Much less code to maintain
- Just one thread for handling HTTP connections versus epee's pool of
threads
- Uses existing LokiMQ job server and existing thread pool for handling
the actual tasks; they are processed/scheduled in the same "rpc" or
"admin" queues as lokimq rpc calls. One notable benefit is that "admin"
rpc commands get their own queue (and thus cannot be delayed by long rpc
commands). Currently the lokimq threads and the http rpc thread pool
and the p2p thread pool and the job queue thread pool and the dns lookup
thread pool and... are *all* different thread pools; this is a step
towards consolidating them.
- Very little mutex contention (which has been a major problem with epee
RPC in the past): there is one mutex (inside uWebSockets) for putting
responses back into the thread managing the connection; everything
internally gets handled through (lock-free) lokimq inproc sockets.
- Faster RPC performance on average, and much better worst case
performance. Epee's http interface seems to have some race condition
that ocassionally stalls a request (even a very simple one) for a dozen
or more seconds for no good reason.
- Long polling gets redone here to no longer need threads; instead we
just store the request and respond when the thread pool, or else in a
timer (that runs once/second) for timing out long polls.
---
The basic idea of how this works from a high level:
We launch a single thread to handle HTTP RPC requests and response data.
This uWebSockets thread is essentially running an event loop: it never
actually handles any logic; it only serves to shuttle data that arrives
in a request to some other thread, and then, at some later point, to
send some reply back to that waiting connection. Everything is
asynchronous and non-blocking here: the basic uWebSockets event loop
just operates as things arrive, passes it off immediately, and goes back
to waiting for the next thing to arrive.
The basic flow is like this:
0. uWS thread -- listens on localhost:22023
1. uWS thread -- incoming request on localhost:22023
2. uWS thread -- fires callback, which injects the task into the LokiMQ job queue
3. LMQ main loop -- schedules it as an RPC job
4. LMQ rpc thread -- Some LokiMQ thread runs it, gets the result
5. LMQ rpc thread -- Result gets queued up for the uWS thread
6. uWS thread -- takes the request and starts sending it
(asynchronously) back to the requestor.
In more detail:
uWebSockets has registered has registered handlers for non-jsonrpc
requests (legacy JSON or binary). If the port is restricted then admin
commands get mapped to a "Access denied" response handler, otherwise
public commands (and admin commands on an unrestricted port) go to the
rpc command handler.
POST requests to /json_rpc have their own handler; this is a little
different than the above because it has to parse the request before it
can determine whether it is allowed or not, but once this is done it
continues roughly the same as legacy/binary requests.
uWebSockets then listens on the given IP/port for new incoming requests,
and starts listening for requests in a thread (we own this thread).
When a request arrives, it fires the event handler for that request.
(This may happen multiple times, if the client is sending a bunch of
data in a POST request). Once we have the full request, we then queue
the job in LokiMQ, putting it in the "rpc" or "admin" command
categories. (The one practical different here is that "admin" is
configured to be allowed to start up its own thread if all other threads
are busy, while "rpc" commands are prioritized along with everything
else.) LokiMQ then schedules this, along with native LokiMQ "rpc." or
"admin." requests.
When a LMQ worker thread becomes available, the RPC command gets called
in it and runs. Whatever output it produces (or error message, if it
throws) then gets wrapped up in jsonrpc boilerplate (if necessary), and
delivered to the uWebSockets thread to be sent in reply to that request.
uWebSockets picks up the data and sends whatever it can without
blocking, then buffers whatever it couldn't send to be sent again in a
later event loop iteration once the requestor can accept more data.
(This part is outside lokid; we only have to give uWS the data and let
it worry about delivery).
---
PR specifics:
Things removed from this PR:
1. ssl settings; with this PR the HTTP RPC interface is plain-text. The
previous default generated a self-signed certificate for the server on
startup and then the client accepted any certificate. This is actually
*worse* than unencrypted because it is entirely MITM-readable and yet
might make people think that their RPC communication is encrypted, and
setting up actual certificates is difficult enough that I think most
people don't bother.
uWebSockets *does* support HTTPS, and we could glue the existing options
into it, but I'm not convinced it's worthwhile: it works much better to
put HTTPS in a front-end proxy holding the certificate that proxies
requests to the backend (which can then listen in restricted mode on
some localhost port). One reason this is better is that it is much
easier to reload and/or restart such a front-end server, while
certificate updates with lokid require a full restart. Another reason
is that you get an error page instead of a timeout if something is wrong
with the backend. Finally we also save having to generate a temporary
certificate on *every* lokid invocation.
2. HTTP Digest authentication. Digest authentication is obsolete (and
was already obsolete when it got added to Monero). HTTP-Digest was
originally an attempt to provide a password authentication mechanism
that does not leak the password in transit, but still required that the
server know the password. It only has marginal value against replay
attacks, and is made entirely obsolete by sending traffic over HTTPS
instead. No client out there supports Digest but *not* Basic auth, and
so given the limited usefulness it seems pointless to support more than
Basic auth for HTTP RPC login.
What's worse is that epee's HTTP Digest authentication is a terrible
implementation: it uses boost::spirit -- a recursive descent parser
meant for building complex language grammars -- just to parse a single
HTTP header for Digest auth. This is a big load of crap that should
never have been accepted upstream, and that we should get rid of (even
if we wanted to support Digest auth it takes less than 100 lines of code
to do it when *not* using a recursive descent parser).
2020-06-29 01:23:06 +02:00
|
|
|
# uSockets doesn't really have a proper build system (just a very simple Makefile) so build it
|
|
|
|
|
# ourselves.
|
|
|
|
|
if (NOT CMAKE_VERSION VERSION_LESS 3.12)
|
|
|
|
|
set(conf_depends "CONFIGURE_DEPENDS")
|
|
|
|
|
else()
|
|
|
|
|
set(conf_depends "")
|
|
|
|
|
endif()
|
|
|
|
|
file(GLOB usockets_src ${conf_depends} uWebSockets/uSockets/src/*.c uWebSockets/uSockets/src/eventing/*.c)
|
|
|
|
|
file(COPY uWebSockets/uSockets/src/libusockets.h DESTINATION uWebSockets)
|
|
|
|
|
add_library(uSockets STATIC EXCLUDE_FROM_ALL ${usockets_src})
|
|
|
|
|
target_compile_definitions(uSockets PRIVATE LIBUS_NO_SSL=1)
|
|
|
|
|
target_include_directories(uSockets PRIVATE uWebSockets/uSockets/src)
|
|
|
|
|
|
2020-10-20 22:02:45 +02:00
|
|
|
# On Windows uSockets uses libuv for its event loop; on Mac kqueue is the default, but that seems to
|
|
|
|
|
# not be reliable on older macos versions (like 10.12), so we use libuv on macos as well.
|
2021-04-22 03:03:19 +02:00
|
|
|
if (WIN32 OR (APPLE AND NOT IOS))
|
2020-07-17 03:25:17 +02:00
|
|
|
add_subdirectory(libuv EXCLUDE_FROM_ALL)
|
|
|
|
|
target_link_libraries(uSockets uv_a)
|
2020-10-20 22:02:45 +02:00
|
|
|
target_compile_definitions(uSockets PUBLIC LIBUS_USE_LIBUV)
|
2020-07-17 03:25:17 +02:00
|
|
|
endif()
|
|
|
|
|
|
Replace epee http rpc server with uWebSockets
This replaces the NIH epee http server which does not work all that well
with an external C++ library called uWebSockets. Fundamentally this
gives the following advantages:
- Much less code to maintain
- Just one thread for handling HTTP connections versus epee's pool of
threads
- Uses existing LokiMQ job server and existing thread pool for handling
the actual tasks; they are processed/scheduled in the same "rpc" or
"admin" queues as lokimq rpc calls. One notable benefit is that "admin"
rpc commands get their own queue (and thus cannot be delayed by long rpc
commands). Currently the lokimq threads and the http rpc thread pool
and the p2p thread pool and the job queue thread pool and the dns lookup
thread pool and... are *all* different thread pools; this is a step
towards consolidating them.
- Very little mutex contention (which has been a major problem with epee
RPC in the past): there is one mutex (inside uWebSockets) for putting
responses back into the thread managing the connection; everything
internally gets handled through (lock-free) lokimq inproc sockets.
- Faster RPC performance on average, and much better worst case
performance. Epee's http interface seems to have some race condition
that ocassionally stalls a request (even a very simple one) for a dozen
or more seconds for no good reason.
- Long polling gets redone here to no longer need threads; instead we
just store the request and respond when the thread pool, or else in a
timer (that runs once/second) for timing out long polls.
---
The basic idea of how this works from a high level:
We launch a single thread to handle HTTP RPC requests and response data.
This uWebSockets thread is essentially running an event loop: it never
actually handles any logic; it only serves to shuttle data that arrives
in a request to some other thread, and then, at some later point, to
send some reply back to that waiting connection. Everything is
asynchronous and non-blocking here: the basic uWebSockets event loop
just operates as things arrive, passes it off immediately, and goes back
to waiting for the next thing to arrive.
The basic flow is like this:
0. uWS thread -- listens on localhost:22023
1. uWS thread -- incoming request on localhost:22023
2. uWS thread -- fires callback, which injects the task into the LokiMQ job queue
3. LMQ main loop -- schedules it as an RPC job
4. LMQ rpc thread -- Some LokiMQ thread runs it, gets the result
5. LMQ rpc thread -- Result gets queued up for the uWS thread
6. uWS thread -- takes the request and starts sending it
(asynchronously) back to the requestor.
In more detail:
uWebSockets has registered has registered handlers for non-jsonrpc
requests (legacy JSON or binary). If the port is restricted then admin
commands get mapped to a "Access denied" response handler, otherwise
public commands (and admin commands on an unrestricted port) go to the
rpc command handler.
POST requests to /json_rpc have their own handler; this is a little
different than the above because it has to parse the request before it
can determine whether it is allowed or not, but once this is done it
continues roughly the same as legacy/binary requests.
uWebSockets then listens on the given IP/port for new incoming requests,
and starts listening for requests in a thread (we own this thread).
When a request arrives, it fires the event handler for that request.
(This may happen multiple times, if the client is sending a bunch of
data in a POST request). Once we have the full request, we then queue
the job in LokiMQ, putting it in the "rpc" or "admin" command
categories. (The one practical different here is that "admin" is
configured to be allowed to start up its own thread if all other threads
are busy, while "rpc" commands are prioritized along with everything
else.) LokiMQ then schedules this, along with native LokiMQ "rpc." or
"admin." requests.
When a LMQ worker thread becomes available, the RPC command gets called
in it and runs. Whatever output it produces (or error message, if it
throws) then gets wrapped up in jsonrpc boilerplate (if necessary), and
delivered to the uWebSockets thread to be sent in reply to that request.
uWebSockets picks up the data and sends whatever it can without
blocking, then buffers whatever it couldn't send to be sent again in a
later event loop iteration once the requestor can accept more data.
(This part is outside lokid; we only have to give uWS the data and let
it worry about delivery).
---
PR specifics:
Things removed from this PR:
1. ssl settings; with this PR the HTTP RPC interface is plain-text. The
previous default generated a self-signed certificate for the server on
startup and then the client accepted any certificate. This is actually
*worse* than unencrypted because it is entirely MITM-readable and yet
might make people think that their RPC communication is encrypted, and
setting up actual certificates is difficult enough that I think most
people don't bother.
uWebSockets *does* support HTTPS, and we could glue the existing options
into it, but I'm not convinced it's worthwhile: it works much better to
put HTTPS in a front-end proxy holding the certificate that proxies
requests to the backend (which can then listen in restricted mode on
some localhost port). One reason this is better is that it is much
easier to reload and/or restart such a front-end server, while
certificate updates with lokid require a full restart. Another reason
is that you get an error page instead of a timeout if something is wrong
with the backend. Finally we also save having to generate a temporary
certificate on *every* lokid invocation.
2. HTTP Digest authentication. Digest authentication is obsolete (and
was already obsolete when it got added to Monero). HTTP-Digest was
originally an attempt to provide a password authentication mechanism
that does not leak the password in transit, but still required that the
server know the password. It only has marginal value against replay
attacks, and is made entirely obsolete by sending traffic over HTTPS
instead. No client out there supports Digest but *not* Basic auth, and
so given the limited usefulness it seems pointless to support more than
Basic auth for HTTP RPC login.
What's worse is that epee's HTTP Digest authentication is a terrible
implementation: it uses boost::spirit -- a recursive descent parser
meant for building complex language grammars -- just to parse a single
HTTP header for Digest auth. This is a big load of crap that should
never have been accepted upstream, and that we should get rid of (even
if we wanted to support Digest auth it takes less than 100 lines of code
to do it when *not* using a recursive descent parser).
2020-06-29 01:23:06 +02:00
|
|
|
|
|
|
|
|
# The uWebSockets C++ layer is header-only but isn't actually prefixed in the repository itself, but
|
|
|
|
|
# rather only on install (which, as above, is just a very simple Makefile). This is unfortunate
|
|
|
|
|
# because it means that we can't use `#include <uWebSockets/App.h>` directly with the repo; so
|
|
|
|
|
# instead we emulate the installation process into the build directory and include it (with the
|
|
|
|
|
# prefix) from there.
|
|
|
|
|
file(COPY uWebSockets/src/ DESTINATION uWebSockets/uWebSockets FILES_MATCHING PATTERN "*.h" PATTERN "*.hpp")
|
|
|
|
|
add_library(uWebSockets INTERFACE)
|
|
|
|
|
target_include_directories(uWebSockets INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/uWebSockets)
|
|
|
|
|
target_link_libraries(uWebSockets INTERFACE uSockets)
|
|
|
|
|
target_compile_definitions(uWebSockets INTERFACE UWS_HTTPRESPONSE_NO_WRITEMARK UWS_NO_ZLIB)
|
Replace epee http client with curl-based client
In short: epee's http client is garbage, standard violating, and
unreliable.
This completely removes the epee http client support and replaces it
with cpr, a curl-based C++ wrapper. rpc/http_client.h wraps cpr for RPC
requests specifically, but it is also usable directly.
This replacement has a number of advantages:
- requests are considerably more reliable. The epee http client code
assumes that a connection will be kept alive forever, and returns a
failure if a connection is ever closed. This results in some very
annoying things: for example, preparing a transaction and then waiting
a long tim before confirming it will usually result in an error
communication with the daemon. This is just terribly behaviour: the
right thing to do on a connection failure is to resubmit the request.
- epee's http client is broken in lots of other ways: for example, it
tries throwing SSL at the port to see if it is HTTPS, but this is
protocol violating and just breaks (with a several second timeout) on
anything that *isn't* epee http server (for example, when lokid is
behind a proxying server).
- even when it isn't doing the above, the client breaks in other ways:
for example, there is a comment (replaced in this PR) in the Trezor PR
code that forces a connection close after every request because epee's
http client doesn't do proper keep-alive request handling.
- it seems noticeably faster to me in practical use in this PR; both
simple requests (for example, when running `lokid status`) and
wallet<->daemon connections are faster, probably because of crappy
code in epee. (I think this is also related to the throw-ssl-at-it
junk above: the epee client always generates an ssl certificate during
static initialization because it might need one at some point).
- significantly reduces the amount of code we have to maintain.
- removes all the epee ssl option code: curl can handle all of that just
fine.
- removes the epee socks proxy code; curl can handle that just fine.
(And can do more: it also supports using HTTP/HTTPS proxies).
- When a cli wallet connection fails we know show why it failed (which
now is an error message from curl), which could have all sorts of
reasons like hostname resolution failure, bad ssl certificate, etc.
Previously you just got a useless generic error that tells you
nothing.
Other related changes in this PR:
- Drops the check-for-update and download-update code. To the best of
my knowledge these have never been supported in loki-core and so it
didn't seem worth the trouble to convert them to use cpr for the
requests.
- Cleaned up node_rpc_proxy return values: there was an inconsistent mix
of ways to return errors and how the returned strings were handled.
Instead this cleans it up to return a pair<bool, val>, which (with
C++17) can be transparently captured as:
auto [success, val] = node.whatever(req);
This drops the failure message string, but it was almost always set to
something fairly useless (if we want to resurrect it we could easily
change the first element to be a custom type with a bool operator for
success, and a `.error` attribute containing some error string, but
for the most part the current code wasn't doing much useful with the
failure string).
- changed local detection (for automatic trusted daemon determination)
to just look for localhost, and to not try to resolve anything.
Trusting non-public IPs does not work well (e.g. with lokinet where
all .loki addresses resolve to a local IP).
- ssl fingerprint option is removed; this isn't supported by curl
(because it is essentially just duplicating what a custom cainfo
bundle does)
- --daemon-ssl-allow-chained is removed; it wasn't a useful option (if
you don't want chaining, don't specify a cainfo chain).
- --daemon-address is now a URL instead of just host:port. (If you omit
the protocol, http:// is prepended).
- --daemon-host and --daemon-port are now deprecated and produce a
warning (in simplewallet) if used; the replacement is to use
--daemon-address.
- --daemon-ssl is deprecated; specify --daemon-address=https://whatever
instead.
- the above three are now hidden from --help
- reordered the wallet connection options to make more logical sense.
2020-07-26 22:29:49 +02:00
|
|
|
|
|
|
|
|
|
|
|
|
|
# cpr configuration. Ideally we'd just do this via add_subdirectory, but cpr's cmake requires
|
|
|
|
|
# 3.15+, and we target lower than that (and this is fairly simple to build).
|
|
|
|
|
|
2020-08-04 00:15:51 +02:00
|
|
|
if(NOT BUILD_STATIC_DEPS)
|
2020-08-07 22:35:13 +02:00
|
|
|
find_package(CURL REQUIRED COMPONENTS HTTP HTTPS SSL)
|
|
|
|
|
|
|
|
|
|
# CURL::libcurl wasn't added to FindCURL until cmake 3.12, so add it if necessary
|
|
|
|
|
if (CMAKE_VERSION VERSION_LESS 3.12 AND NOT TARGET CURL::libcurl)
|
2020-08-10 02:17:30 +02:00
|
|
|
add_library(libcurl UNKNOWN IMPORTED GLOBAL)
|
|
|
|
|
set_target_properties(libcurl PROPERTIES
|
|
|
|
|
IMPORTED_LOCATION ${CURL_LIBRARIES}
|
|
|
|
|
INTERFACE_INCLUDE_DIRECTORIES "${CURL_INCLUDE_DIRS}")
|
|
|
|
|
add_library(CURL_libcurl INTERFACE)
|
|
|
|
|
target_link_libraries(CURL_libcurl INTERFACE libcurl)
|
|
|
|
|
add_library(CURL::libcurl ALIAS CURL_libcurl)
|
2020-08-07 22:35:13 +02:00
|
|
|
endif()
|
2020-08-04 00:15:51 +02:00
|
|
|
endif()
|
Replace epee http client with curl-based client
In short: epee's http client is garbage, standard violating, and
unreliable.
This completely removes the epee http client support and replaces it
with cpr, a curl-based C++ wrapper. rpc/http_client.h wraps cpr for RPC
requests specifically, but it is also usable directly.
This replacement has a number of advantages:
- requests are considerably more reliable. The epee http client code
assumes that a connection will be kept alive forever, and returns a
failure if a connection is ever closed. This results in some very
annoying things: for example, preparing a transaction and then waiting
a long tim before confirming it will usually result in an error
communication with the daemon. This is just terribly behaviour: the
right thing to do on a connection failure is to resubmit the request.
- epee's http client is broken in lots of other ways: for example, it
tries throwing SSL at the port to see if it is HTTPS, but this is
protocol violating and just breaks (with a several second timeout) on
anything that *isn't* epee http server (for example, when lokid is
behind a proxying server).
- even when it isn't doing the above, the client breaks in other ways:
for example, there is a comment (replaced in this PR) in the Trezor PR
code that forces a connection close after every request because epee's
http client doesn't do proper keep-alive request handling.
- it seems noticeably faster to me in practical use in this PR; both
simple requests (for example, when running `lokid status`) and
wallet<->daemon connections are faster, probably because of crappy
code in epee. (I think this is also related to the throw-ssl-at-it
junk above: the epee client always generates an ssl certificate during
static initialization because it might need one at some point).
- significantly reduces the amount of code we have to maintain.
- removes all the epee ssl option code: curl can handle all of that just
fine.
- removes the epee socks proxy code; curl can handle that just fine.
(And can do more: it also supports using HTTP/HTTPS proxies).
- When a cli wallet connection fails we know show why it failed (which
now is an error message from curl), which could have all sorts of
reasons like hostname resolution failure, bad ssl certificate, etc.
Previously you just got a useless generic error that tells you
nothing.
Other related changes in this PR:
- Drops the check-for-update and download-update code. To the best of
my knowledge these have never been supported in loki-core and so it
didn't seem worth the trouble to convert them to use cpr for the
requests.
- Cleaned up node_rpc_proxy return values: there was an inconsistent mix
of ways to return errors and how the returned strings were handled.
Instead this cleans it up to return a pair<bool, val>, which (with
C++17) can be transparently captured as:
auto [success, val] = node.whatever(req);
This drops the failure message string, but it was almost always set to
something fairly useless (if we want to resurrect it we could easily
change the first element to be a custom type with a bool operator for
success, and a `.error` attribute containing some error string, but
for the most part the current code wasn't doing much useful with the
failure string).
- changed local detection (for automatic trusted daemon determination)
to just look for localhost, and to not try to resolve anything.
Trusting non-public IPs does not work well (e.g. with lokinet where
all .loki addresses resolve to a local IP).
- ssl fingerprint option is removed; this isn't supported by curl
(because it is essentially just duplicating what a custom cainfo
bundle does)
- --daemon-ssl-allow-chained is removed; it wasn't a useful option (if
you don't want chaining, don't specify a cainfo chain).
- --daemon-address is now a URL instead of just host:port. (If you omit
the protocol, http:// is prepended).
- --daemon-host and --daemon-port are now deprecated and produce a
warning (in simplewallet) if used; the replacement is to use
--daemon-address.
- --daemon-ssl is deprecated; specify --daemon-address=https://whatever
instead.
- the above three are now hidden from --help
- reordered the wallet connection options to make more logical sense.
2020-07-26 22:29:49 +02:00
|
|
|
|
|
|
|
|
file(GLOB cpr_sources ${conf_depends} cpr/cpr/*.cpp)
|
|
|
|
|
|
|
|
|
|
add_library(cpr STATIC EXCLUDE_FROM_ALL ${cpr_sources})
|
|
|
|
|
target_link_libraries(cpr PUBLIC CURL::libcurl)
|
|
|
|
|
target_include_directories(cpr PUBLIC cpr/include)
|
|
|
|
|
target_compile_definitions(cpr PUBLIC CPR_CURL_NOSIGNAL)
|
|
|
|
|
add_library(cpr::cpr ALIAS cpr)
|
2021-10-05 02:33:24 +02:00
|
|
|
|
2022-04-14 01:31:46 +02:00
|
|
|
file(READ cpr/CMakeLists.txt cpr_cmake_head LIMIT 1000)
|
|
|
|
|
if(cpr_cmake_head MATCHES "project\\(cpr VERSION ([0-9]+)\.([0-9]+)\.([0-9]+) LANGUAGES CXX\\)")
|
|
|
|
|
set(cpr_VERSION_MAJOR ${CMAKE_MATCH_1})
|
|
|
|
|
set(cpr_VERSION_MINOR ${CMAKE_MATCH_2})
|
|
|
|
|
set(cpr_VERSION_PATCH ${CMAKE_MATCH_3})
|
|
|
|
|
set(cpr_VERSION "${cpr_VERSION_MAJOR}.${cpr_VERSION_MINOR}.${cpr_VERSION_PATCH}")
|
|
|
|
|
set(cpr_VERSION_NUM "(${cpr_VERSION_MAJOR} * 0x10000 + ${cpr_VERSION_MINOR} * 0x100 + ${cpr_VERSION_PATCH})")
|
|
|
|
|
|
|
|
|
|
configure_file(cpr/cmake/cprver.h.in "${CMAKE_CURRENT_BINARY_DIR}/cpr_generated_includes/cpr/cprver.h")
|
|
|
|
|
target_include_directories(cpr PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/cpr_generated_includes")
|
|
|
|
|
else()
|
|
|
|
|
message(FATAL_ERROR "Could not identify cpr submodule version!")
|
|
|
|
|
endif()
|
|
|
|
|
|
|
|
|
|
|
2021-10-07 01:04:23 +02:00
|
|
|
# Hack around SQLiteCpp's attempts to locate sqlite3 because we *don't* want to link against the
|
|
|
|
|
# system one, but don't download and build the embedded one until build time. Thankfully it
|
|
|
|
|
# actually links against the SQLite::SQLite3 cmake target if it already exists, so all we have to do
|
|
|
|
|
# is set that up and circumvent some of the non-target bits of its FindSQLite3.cmake.
|
|
|
|
|
set(SQLite3_FOUND TRUE CACHE BOOL "" FORCE)
|
|
|
|
|
file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/ignored")
|
|
|
|
|
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/ignored/sqlite3.h" "#define SQLITE_VERSION \"${SQLite3_VERSION}\"")
|
|
|
|
|
set(SQLite3_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/ignored" CACHE STRING "" FORCE)
|
|
|
|
|
set(SQLite3_LIBRARY "ignored" CACHE STRING "" FORCE)
|
|
|
|
|
set(SQLITECPP_INTERNAL_SQLITE OFF CACHE BOOL "don't build SQLiteCpp's internal sqlite3" FORCE)
|
|
|
|
|
set(SQLITE_ENABLE_COLUMN_METADATA OFF CACHE BOOL "" FORCE)
|
|
|
|
|
set(SQLITECPP_RUN_CPPLINT OFF CACHE BOOL "" FORCE)
|
|
|
|
|
set(SQLITECPP_RUN_CPPCHECK OFF CACHE BOOL "" FORCE)
|
|
|
|
|
set(SQLITECPP_RUN_DOXYGEN OFF CACHE BOOL "" FORCE)
|
|
|
|
|
set(SQLITECPP_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
|
|
|
|
|
set(SQLITECPP_BUILD_TESTS OFF CACHE BOOL "" FORCE)
|
|
|
|
|
|
2021-10-05 02:33:24 +02:00
|
|
|
add_subdirectory(SQLiteCpp)
|
2021-07-19 20:42:38 +02:00
|
|
|
|
|
|
|
|
add_subdirectory(Catch2)
|
2022-10-23 22:08:52 +02:00
|
|
|
|
2022-12-05 21:33:56 +01:00
|
|
|
if(BUILD_PYBIND)
|
|
|
|
|
add_subdirectory(pybind11 EXCLUDE_FROM_ALL)
|
|
|
|
|
endif()
|
2022-10-23 22:08:52 +02:00
|
|
|
|
