oxen-mq/CMakeLists.txt

239 lines
7.7 KiB
CMake
Raw Permalink Normal View History

set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
find_program(CCACHE_PROGRAM ccache)
if(CCACHE_PROGRAM)
foreach(lang C CXX)
if(NOT DEFINED CMAKE_${lang}_COMPILER_LAUNCHER AND NOT CMAKE_${lang}_COMPILER MATCHES ".*/ccache")
message(STATUS "Enabling ccache for ${lang}")
set(CMAKE_${lang}_COMPILER_LAUNCHER ${CCACHE_PROGRAM} CACHE STRING "")
endif()
endforeach()
endif()
cmake_minimum_required(VERSION 3.7)
# Has to be set before `project()`, and ignored on non-macos:
set(CMAKE_OSX_DEPLOYMENT_TARGET 10.12 CACHE STRING "macOS deployment target (Apple clang only)")
2021-04-15 20:14:36 +02:00
project(liboxenmq
Add epoll support for Linux Each call to zmq::poll is painfully slow when we have many open zmq sockets, such as when we have 1800 outbound connections (i.e. connected to every other service node, as services nodes might have sometimes and the Session push notification server *always* has). In testing on my local Ryzen 5950 system each time we go back to zmq::poll incurs about 1.5ms of (mostly system) CPU time with 2000 open outbound sockets, and so if we're being pelted with a nearly constant stream of requests (such as happens with the Session push notification server) we incur massive CPU costs every time we finish processing messages and go back to wait (via zmq::poll) for more. In testing a simple ZMQ (no OxenMQ) client/server that establishes 2000 connections to a server, and then has the server send a message back on a random connection every 1ms, we get atrocious CPU usage: the proxy thread spends a constant 100% CPU time. Virtually all of this is in the poll call itself, though, so we aren't really bottlenecked by how much can go through the proxy thread: in such a scenario the poll call uses its CPU then returns right away, we process the queue of messages, and return to another poll call. If we have lots of messages received in that time, though (because messages are coming fast and the poll was slow) then we process a lot all at once before going back to the poll, so the main consequences here are that: 1) We use a huge amount of CPU 2) We introduce latency in a busy situation because the CPU has to make the poll call (e.g. 1.5ms) before the next message can be processed. 3) If traffic is very bursty then the latency can manifest another problem: in the time it takes to poll we could accumulate enough incoming messages to overfill our internal per-category job queue, which was happening in the SPNS. (I also tested with 20k connections, and the poll time scaling was linear: we still processed everything, but in larger chunks because every poll call took about 15ms, and so we'd have about 15 messages at a time to process with added latency of up to 15ms). Switching to epoll *drastically* reduces the CPU usage in two ways: 1) It's massively faster by design: there's a single setup and communication of all the polling details to the kernel which we only have to do when our set of zmq sockets changes (which is relatively rare). 2) We can further reduce CPU time because epoll tells us *which* sockets need attention, and so if only 1 connection out of the 2000 sent us something we can only bother checking that single socket for messages. (In theory we can do the same with zmq::poll by querying for events available on the socket, but in practice it doesn't improve anything over just trying to read from them all). In my straight zmq test script, using epoll instead reduced CPU usage in the sends-every-1ms scenario from a constant pegged 100% of a core to an average of 2-3% of a single core. (Moreover this CPU usage level didn't noticeably change when using 20k connections instead of 2k).
2023-09-14 19:38:39 +02:00
VERSION 1.2.16
2021-04-15 20:14:36 +02:00
LANGUAGES CXX C)
include(GNUInstallDirs)
2021-04-15 20:14:36 +02:00
message(STATUS "oxenmq v${PROJECT_VERSION}")
2021-01-14 19:37:14 +01:00
set(OXENMQ_LIBVERSION 0)
if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
2021-01-14 19:37:14 +01:00
set(oxenmq_IS_TOPLEVEL_PROJECT TRUE)
else()
2021-01-14 19:37:14 +01:00
set(oxenmq_IS_TOPLEVEL_PROJECT FALSE)
endif()
option(BUILD_SHARED_LIBS "Build shared libraries instead of static ones" ON)
2021-01-14 19:37:14 +01:00
set(oxenmq_INSTALL_DEFAULT OFF)
if(BUILD_SHARED_LIBS OR oxenmq_IS_TOPLEVEL_PROJECT)
set(oxenmq_INSTALL_DEFAULT ON)
endif()
Add epoll support for Linux Each call to zmq::poll is painfully slow when we have many open zmq sockets, such as when we have 1800 outbound connections (i.e. connected to every other service node, as services nodes might have sometimes and the Session push notification server *always* has). In testing on my local Ryzen 5950 system each time we go back to zmq::poll incurs about 1.5ms of (mostly system) CPU time with 2000 open outbound sockets, and so if we're being pelted with a nearly constant stream of requests (such as happens with the Session push notification server) we incur massive CPU costs every time we finish processing messages and go back to wait (via zmq::poll) for more. In testing a simple ZMQ (no OxenMQ) client/server that establishes 2000 connections to a server, and then has the server send a message back on a random connection every 1ms, we get atrocious CPU usage: the proxy thread spends a constant 100% CPU time. Virtually all of this is in the poll call itself, though, so we aren't really bottlenecked by how much can go through the proxy thread: in such a scenario the poll call uses its CPU then returns right away, we process the queue of messages, and return to another poll call. If we have lots of messages received in that time, though (because messages are coming fast and the poll was slow) then we process a lot all at once before going back to the poll, so the main consequences here are that: 1) We use a huge amount of CPU 2) We introduce latency in a busy situation because the CPU has to make the poll call (e.g. 1.5ms) before the next message can be processed. 3) If traffic is very bursty then the latency can manifest another problem: in the time it takes to poll we could accumulate enough incoming messages to overfill our internal per-category job queue, which was happening in the SPNS. (I also tested with 20k connections, and the poll time scaling was linear: we still processed everything, but in larger chunks because every poll call took about 15ms, and so we'd have about 15 messages at a time to process with added latency of up to 15ms). Switching to epoll *drastically* reduces the CPU usage in two ways: 1) It's massively faster by design: there's a single setup and communication of all the polling details to the kernel which we only have to do when our set of zmq sockets changes (which is relatively rare). 2) We can further reduce CPU time because epoll tells us *which* sockets need attention, and so if only 1 connection out of the 2000 sent us something we can only bother checking that single socket for messages. (In theory we can do the same with zmq::poll by querying for events available on the socket, but in practice it doesn't improve anything over just trying to read from them all). In my straight zmq test script, using epoll instead reduced CPU usage in the sends-every-1ms scenario from a constant pegged 100% of a core to an average of 2-3% of a single core. (Moreover this CPU usage level didn't noticeably change when using 20k connections instead of 2k).
2023-09-14 19:38:39 +02:00
set(oxenmq_EPOLL_DEFAULT OFF)
if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND NOT CMAKE_CROSSCOMPILING)
set(oxenmq_EPOLL_DEFAULT ON)
endif()
2021-01-14 19:37:14 +01:00
option(OXENMQ_BUILD_TESTS "Building and perform oxenmq tests" ${oxenmq_IS_TOPLEVEL_PROJECT})
option(OXENMQ_INSTALL "Add oxenmq libraries and headers to cmake install target; defaults to ON if BUILD_SHARED_LIBS is enabled or we are the top-level project; OFF for a static subdirectory build" ${oxenmq_INSTALL_DEFAULT})
option(OXENMQ_INSTALL_CPPZMQ "Install cppzmq header with oxenmq/ headers (requires OXENMQ_INSTALL)" ON)
Add epoll support for Linux Each call to zmq::poll is painfully slow when we have many open zmq sockets, such as when we have 1800 outbound connections (i.e. connected to every other service node, as services nodes might have sometimes and the Session push notification server *always* has). In testing on my local Ryzen 5950 system each time we go back to zmq::poll incurs about 1.5ms of (mostly system) CPU time with 2000 open outbound sockets, and so if we're being pelted with a nearly constant stream of requests (such as happens with the Session push notification server) we incur massive CPU costs every time we finish processing messages and go back to wait (via zmq::poll) for more. In testing a simple ZMQ (no OxenMQ) client/server that establishes 2000 connections to a server, and then has the server send a message back on a random connection every 1ms, we get atrocious CPU usage: the proxy thread spends a constant 100% CPU time. Virtually all of this is in the poll call itself, though, so we aren't really bottlenecked by how much can go through the proxy thread: in such a scenario the poll call uses its CPU then returns right away, we process the queue of messages, and return to another poll call. If we have lots of messages received in that time, though (because messages are coming fast and the poll was slow) then we process a lot all at once before going back to the poll, so the main consequences here are that: 1) We use a huge amount of CPU 2) We introduce latency in a busy situation because the CPU has to make the poll call (e.g. 1.5ms) before the next message can be processed. 3) If traffic is very bursty then the latency can manifest another problem: in the time it takes to poll we could accumulate enough incoming messages to overfill our internal per-category job queue, which was happening in the SPNS. (I also tested with 20k connections, and the poll time scaling was linear: we still processed everything, but in larger chunks because every poll call took about 15ms, and so we'd have about 15 messages at a time to process with added latency of up to 15ms). Switching to epoll *drastically* reduces the CPU usage in two ways: 1) It's massively faster by design: there's a single setup and communication of all the polling details to the kernel which we only have to do when our set of zmq sockets changes (which is relatively rare). 2) We can further reduce CPU time because epoll tells us *which* sockets need attention, and so if only 1 connection out of the 2000 sent us something we can only bother checking that single socket for messages. (In theory we can do the same with zmq::poll by querying for events available on the socket, but in practice it doesn't improve anything over just trying to read from them all). In my straight zmq test script, using epoll instead reduced CPU usage in the sends-every-1ms scenario from a constant pegged 100% of a core to an average of 2-3% of a single core. (Moreover this CPU usage level didn't noticeably change when using 20k connections instead of 2k).
2023-09-14 19:38:39 +02:00
option(OXENMQ_USE_EPOLL "Use epoll for socket polling (requires Linux)" ${oxenmq_EPOLL_DEFAULT})
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
2021-01-14 19:37:14 +01:00
configure_file(oxenmq/version.h.in oxenmq/version.h @ONLY)
configure_file(liboxenmq.pc.in liboxenmq.pc @ONLY)
2021-01-14 19:38:14 +01:00
2021-01-14 19:37:14 +01:00
add_library(oxenmq
oxenmq/address.cpp
oxenmq/auth.cpp
oxenmq/connections.cpp
oxenmq/jobs.cpp
oxenmq/oxenmq.cpp
oxenmq/proxy.cpp
oxenmq/worker.cpp
)
2021-01-14 19:37:14 +01:00
set_target_properties(oxenmq PROPERTIES SOVERSION ${OXENMQ_LIBVERSION})
Add epoll support for Linux Each call to zmq::poll is painfully slow when we have many open zmq sockets, such as when we have 1800 outbound connections (i.e. connected to every other service node, as services nodes might have sometimes and the Session push notification server *always* has). In testing on my local Ryzen 5950 system each time we go back to zmq::poll incurs about 1.5ms of (mostly system) CPU time with 2000 open outbound sockets, and so if we're being pelted with a nearly constant stream of requests (such as happens with the Session push notification server) we incur massive CPU costs every time we finish processing messages and go back to wait (via zmq::poll) for more. In testing a simple ZMQ (no OxenMQ) client/server that establishes 2000 connections to a server, and then has the server send a message back on a random connection every 1ms, we get atrocious CPU usage: the proxy thread spends a constant 100% CPU time. Virtually all of this is in the poll call itself, though, so we aren't really bottlenecked by how much can go through the proxy thread: in such a scenario the poll call uses its CPU then returns right away, we process the queue of messages, and return to another poll call. If we have lots of messages received in that time, though (because messages are coming fast and the poll was slow) then we process a lot all at once before going back to the poll, so the main consequences here are that: 1) We use a huge amount of CPU 2) We introduce latency in a busy situation because the CPU has to make the poll call (e.g. 1.5ms) before the next message can be processed. 3) If traffic is very bursty then the latency can manifest another problem: in the time it takes to poll we could accumulate enough incoming messages to overfill our internal per-category job queue, which was happening in the SPNS. (I also tested with 20k connections, and the poll time scaling was linear: we still processed everything, but in larger chunks because every poll call took about 15ms, and so we'd have about 15 messages at a time to process with added latency of up to 15ms). Switching to epoll *drastically* reduces the CPU usage in two ways: 1) It's massively faster by design: there's a single setup and communication of all the polling details to the kernel which we only have to do when our set of zmq sockets changes (which is relatively rare). 2) We can further reduce CPU time because epoll tells us *which* sockets need attention, and so if only 1 connection out of the 2000 sent us something we can only bother checking that single socket for messages. (In theory we can do the same with zmq::poll by querying for events available on the socket, but in practice it doesn't improve anything over just trying to read from them all). In my straight zmq test script, using epoll instead reduced CPU usage in the sends-every-1ms scenario from a constant pegged 100% of a core to an average of 2-3% of a single core. (Moreover this CPU usage level didn't noticeably change when using 20k connections instead of 2k).
2023-09-14 19:38:39 +02:00
if(OXENMQ_USE_EPOLL)
target_compile_definitions(oxenmq PRIVATE OXENMQ_USE_EPOLL)
endif()
2020-03-13 19:27:29 +01:00
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
2021-01-14 19:37:14 +01:00
target_link_libraries(oxenmq PRIVATE Threads::Threads)
2020-03-13 19:27:29 +01:00
if(TARGET oxenc::oxenc)
add_library(_oxenmq_external_oxenc INTERFACE IMPORTED)
target_link_libraries(_oxenmq_external_oxenc INTERFACE oxenc::oxenc)
2022-05-30 18:28:52 +02:00
target_link_libraries(oxenmq PUBLIC _oxenmq_external_oxenc)
message(STATUS "using pre-existing oxenc::oxenc target")
elseif(BUILD_SHARED_LIBS)
include(FindPkgConfig)
pkg_check_modules(oxenc liboxenc IMPORTED_TARGET)
if(oxenc_FOUND)
# Work around cmake bug 22180 (PkgConfig::tgt not set if no flags needed)
if(TARGET PkgConfig::oxenc OR CMAKE_VERSION VERSION_GREATER_EQUAL "3.21")
target_link_libraries(oxenmq PUBLIC PkgConfig::oxenc)
endif()
else()
add_subdirectory(oxen-encoding)
target_link_libraries(oxenmq PUBLIC oxenc::oxenc)
endif()
2022-02-07 19:38:19 +01:00
else()
add_subdirectory(oxen-encoding)
target_link_libraries(oxenmq PUBLIC oxenc::oxenc)
endif()
# libzmq is nearly impossible to link statically from a system-installed static library: it depends
# on a ton of other libraries, some of which are not all statically available. If the caller wants
# to mess with this, so be it: they can set up a libzmq target and we'll use it. Otherwise if they
# asked us to do things statically, don't even try to find a system lib and just build it.
2021-01-14 19:37:14 +01:00
set(oxenmq_build_static_libzmq OFF)
if(TARGET libzmq)
2021-01-14 19:37:14 +01:00
target_link_libraries(oxenmq PUBLIC libzmq)
elseif(BUILD_SHARED_LIBS)
include(FindPkgConfig)
pkg_check_modules(libzmq libzmq>=4.3 IMPORTED_TARGET)
if(libzmq_FOUND)
# Debian sid includes a -isystem in the mit-krb package that, starting with pkg-config 0.29.2,
# breaks cmake's pkgconfig module because it stupidly thinks "-isystem" is a path, so if we find
# -isystem in the include dirs then hack it out.
get_property(zmq_inc TARGET PkgConfig::libzmq PROPERTY INTERFACE_INCLUDE_DIRECTORIES)
list(FIND zmq_inc "-isystem" broken_isystem)
if(NOT broken_isystem EQUAL -1)
list(REMOVE_AT zmq_inc ${broken_isystem})
set_property(TARGET PkgConfig::libzmq PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${zmq_inc})
endif()
2021-01-14 19:37:14 +01:00
target_link_libraries(oxenmq PUBLIC PkgConfig::libzmq)
else()
2021-01-14 19:37:14 +01:00
set(oxenmq_build_static_libzmq ON)
endif()
else()
2021-01-14 19:37:14 +01:00
set(oxenmq_build_static_libzmq ON)
endif()
2021-01-14 19:37:14 +01:00
if(oxenmq_build_static_libzmq)
2020-09-16 16:32:49 +02:00
message(STATUS "libzmq >= 4.3 not found or static build requested, building bundled version")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/local-libzmq")
include(LocalLibzmq)
2021-01-14 19:37:14 +01:00
target_link_libraries(oxenmq PUBLIC libzmq_vendor)
endif()
2021-01-14 19:37:14 +01:00
target_include_directories(oxenmq
PUBLIC
$<INSTALL_INTERFACE:>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cppzmq>
)
target_compile_options(oxenmq PRIVATE -Wall -Wextra)
option(WARNINGS_AS_ERRORS "treat all warnings as errors" ON)
if(WARNINGS_AS_ERRORS)
target_compile_options(oxenmq PRIVATE -Werror)
endif()
set_target_properties(oxenmq PROPERTIES
POSITION_INDEPENDENT_CODE ON
CXX_STANDARD 17
CXX_STANDARD_REQUIRED ON
CXX_EXTENSIONS OFF
)
function(link_dep_libs target linktype libdirs)
foreach(lib ${ARGN})
find_library(link_lib-${lib} NAMES ${lib} PATHS ${libdirs})
if(link_lib-${lib})
target_link_libraries(${target} ${linktype} ${link_lib-${lib}})
endif()
endforeach()
endfunction()
# If the caller has already set up a sodium target then we will just link to it, otherwise we go
# looking for it.
if(TARGET sodium)
2021-01-14 19:37:14 +01:00
target_link_libraries(oxenmq PUBLIC sodium)
if(oxenmq_build_static_libzmq)
target_link_libraries(libzmq_vendor INTERFACE sodium)
endif()
else()
include(FindPkgConfig)
pkg_check_modules(sodium REQUIRED libsodium IMPORTED_TARGET)
if(BUILD_SHARED_LIBS)
2021-01-14 19:37:14 +01:00
target_link_libraries(oxenmq PUBLIC PkgConfig::sodium)
if(oxenmq_build_static_libzmq)
target_link_libraries(libzmq_vendor INTERFACE PkgConfig::sodium)
endif()
else()
2021-01-14 19:37:14 +01:00
link_dep_libs(oxenmq PUBLIC "${sodium_STATIC_LIBRARY_DIRS}" ${sodium_STATIC_LIBRARIES})
target_include_directories(oxenmq PUBLIC ${sodium_STATIC_INCLUDE_DIRS})
if(oxenmq_build_static_libzmq)
link_dep_libs(libzmq_vendor INTERFACE "${sodium_STATIC_LIBRARY_DIRS}" ${sodium_STATIC_LIBRARIES})
target_link_libraries(libzmq_vendor INTERFACE ${sodium_STATIC_INCLUDE_DIRS})
endif()
endif()
endif()
2021-01-14 19:37:14 +01:00
add_library(oxenmq::oxenmq ALIAS oxenmq)
export(
2021-01-14 19:37:14 +01:00
TARGETS oxenmq
NAMESPACE oxenmq::
FILE oxenmqTargets.cmake
)
2021-01-14 19:37:14 +01:00
if(OXENMQ_INSTALL)
install(
2021-01-14 19:37:14 +01:00
TARGETS oxenmq
EXPORT oxenmqConfig
DESTINATION ${CMAKE_INSTALL_LIBDIR}
)
install(
2021-01-14 19:37:14 +01:00
FILES oxenmq/address.h
oxenmq/auth.h
oxenmq/batch.h
oxenmq/connections.h
oxenmq/fmt.h
2021-01-14 19:37:14 +01:00
oxenmq/message.h
oxenmq/oxenmq.h
oxenmq/pubsub.h
2021-01-14 19:37:14 +01:00
${CMAKE_CURRENT_BINARY_DIR}/oxenmq/version.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/oxenmq
)
2021-01-14 19:37:14 +01:00
if(OXENMQ_INSTALL_CPPZMQ)
install(
FILES cppzmq/zmq.hpp
2021-01-14 19:37:14 +01:00
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/oxenmq
)
endif()
2020-03-13 19:31:43 +01:00
install(
2021-01-14 19:37:14 +01:00
FILES ${CMAKE_CURRENT_BINARY_DIR}/liboxenmq.pc
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig
)
2021-01-14 19:38:14 +01:00
endif()
2021-01-14 19:37:14 +01:00
if(OXENMQ_BUILD_TESTS)
add_subdirectory(tests)
endif()