From 4597f5ca6c47f22b500bcefa7c822a707d25354d Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Mon, 1 Feb 2021 10:48:27 -0500 Subject: [PATCH] loki -> oxen refactor * rename all namespaces so it compiles again --- .gitmodules | 6 +- CMakeLists.txt | 12 +- examples/client.py | 16 +- examples/server.py | 4 +- examples/test.py | 4 +- lokinet/auth/__main__.py | 14 +- pylokimq/CMakeLists.txt | 10 -- pylokimq/module.cpp | 7 - pyoxenmq/CMakeLists.txt | 10 ++ {pylokimq => pyoxenmq}/bencode.cpp | 6 +- {pylokimq => pyoxenmq}/common.hpp | 4 +- {pylokimq => pyoxenmq}/lokimq.cpp | 63 +++---- pyoxenmq/module.cpp | 7 + pyoxenmq/oxenmq.cpp | 255 +++++++++++++++++++++++++++++ setup.py | 8 +- 15 files changed, 342 insertions(+), 84 deletions(-) delete mode 100644 pylokimq/CMakeLists.txt delete mode 100644 pylokimq/module.cpp create mode 100644 pyoxenmq/CMakeLists.txt rename {pylokimq => pyoxenmq}/bencode.cpp (76%) rename {pylokimq => pyoxenmq}/common.hpp (78%) rename {pylokimq => pyoxenmq}/lokimq.cpp (85%) create mode 100644 pyoxenmq/module.cpp create mode 100644 pyoxenmq/oxenmq.cpp diff --git a/.gitmodules b/.gitmodules index f09e1dd..bf09f07 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,7 +2,7 @@ path = external/pybind11 url = https://github.com/pybind/pybind11 branch = stable -[submodule "external/loki-mq"] - path = external/loki-mq - url = https://github.com/loki-project/loki-mq +[submodule "external/oxen-mq"] + path = external/oxen-mq + url = https://github.com/oxen-io/loki-mq branch = dev diff --git a/CMakeLists.txt b/CMakeLists.txt index 3fb573a..0b557af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,10 +8,10 @@ if(CCACHE_PROGRAM) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}") endif() -set(PROJECT_NAME pylokimq) +set(PROJECT_NAME pyoxenmq) project(${PROJECT_NAME} - VERSION 0.0.1 - DESCRIPTION "pybind layer for lokimq" + VERSION 0.1.0 + DESCRIPTION "pybind layer for oxenmq" LANGUAGES CXX) @@ -41,7 +41,7 @@ if(SUBMODULE_CHECK) endfunction () check_submodule(external/pybind11) - check_submodule(external/loki-mq) + check_submodule(external/oxen-mq) endif() endif() @@ -50,6 +50,6 @@ add_compile_options(-Wno-deprecated-declarations) add_subdirectory(external/pybind11) include(FindPkgConfig) -pkg_check_modules(lokimq REQUIRED IMPORTED_TARGET GLOBAL liblokimq) +pkg_check_modules(oxenmq REQUIRED IMPORTED_TARGET GLOBAL liboxenmq) -add_subdirectory(pylokimq) +add_subdirectory(pyoxenmq) diff --git a/examples/client.py b/examples/client.py index dfbea2d..e551c8e 100644 --- a/examples/client.py +++ b/examples/client.py @@ -1,20 +1,20 @@ -import pylokimq +import pyoxenmq -def do_connected(lmq, conn): +def do_connected(mq, conn): print("connected via", conn) - return lmq.request(conn, "llarp.auth", ["dq3j4dj99w6wi4t4yjnya8sxtqr1rojt8jgnn6467o6aoenm3o3o.loki", "5:token"]) + return mq.request(conn, "llarp.auth", ["dq3j4dj99w6wi4t4yjnya8sxtqr1rojt8jgnn6467o6aoenm3o3o.loki", "5:token"]) -def do_request(lmq): +def do_request(mq): print('connect') - conn = lmq.connect_remote("ipc:///tmp/lmq.sock") + conn = mq.connect_remote("ipc:///tmp/lmq.sock") if conn: return do_connected(lmq, conn) def main(): - lmq = pylokimq.LokiMQ() + mq = pyoxenmq.OxenMQ() print("start") - lmq.start() - print(do_request(lmq)) + mq.start() + print(do_request(mq)) print("done") if __name__ == '__main__': diff --git a/examples/server.py b/examples/server.py index 073df13..f468ed2 100644 --- a/examples/server.py +++ b/examples/server.py @@ -1,4 +1,4 @@ -import pylokimq +import pyoxenmq import time def handle_auth(args): @@ -7,7 +7,7 @@ def handle_auth(args): return "OK" def main(): - lmq = pylokimq.LokiMQ() + lmq = pyoxenmq.OxenMQ() lmq.listen_plain("ipc:///tmp/lmq.sock") lmq.add_anonymous_category("llarp") lmq.add_request_command("llarp", "auth", handle_auth) diff --git a/examples/test.py b/examples/test.py index 9d5858c..22895cd 100644 --- a/examples/test.py +++ b/examples/test.py @@ -1,11 +1,11 @@ -import pylokimq +import pyoxenmq import time def handle_ping(args): print(args) return args -lmq = pylokimq.LokiMQ() +lmq = pyoxenmq.OxenMQ() lmq.listen_plain("ipc:///tmp/lmq.sock") lmq.add_anonymous_category("python") lmq.add_request_command("python", "ping", handle_ping) diff --git a/lokinet/auth/__main__.py b/lokinet/auth/__main__.py index 32e3795..8685e3f 100644 --- a/lokinet/auth/__main__.py +++ b/lokinet/auth/__main__.py @@ -1,4 +1,4 @@ -import pylokimq +import pyoxenmq import base64 import subprocess import shlex @@ -61,7 +61,7 @@ def decode_value(data, first=None): def decode_address(data): - return '{}.loki'.format(pylokimq.base32z_encode(decode_value(data)[b's'][b's'])) + return '{}.loki'.format(pyoxenmq.base32z_encode(decode_value(data)[b's'][b's'])) def handle_auth_impl(args, cmd): cmd2 = cmd @@ -85,11 +85,11 @@ def main(): ap.add_argument("--cmd", required=True, help="script to call for authentication") args = ap.parse_args() cmd = shlex.split(args.cmd) - lmq = pylokimq.LokiMQ() - lmq.listen_plain(args.bind) - lmq.add_anonymous_category("llarp") - lmq.add_request_command("llarp", "auth", lambda x : handle_auth(x, cmd)) - lmq.start() + mq = pyoxenmq.OxenMQ() + mq.listen_plain(args.bind) + mq.add_anonymous_category("llarp") + mq.add_request_command("llarp", "auth", lambda x : handle_auth(x, cmd)) + mq.start() print("server started") while True: time.sleep(1) diff --git a/pylokimq/CMakeLists.txt b/pylokimq/CMakeLists.txt deleted file mode 100644 index d5f39d0..0000000 --- a/pylokimq/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ - -pybind11_add_module(pylokimq MODULE - bencode.cpp - lokimq.cpp - module.cpp -) - -target_link_libraries(pylokimq PUBLIC PkgConfig::lokimq) - - diff --git a/pylokimq/module.cpp b/pylokimq/module.cpp deleted file mode 100644 index 8cebe6e..0000000 --- a/pylokimq/module.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "common.hpp" - -PYBIND11_MODULE(pylokimq, m) -{ - lokimq::LokiMQ_Init(m); - lokimq::BEncode_Init(m); -} diff --git a/pyoxenmq/CMakeLists.txt b/pyoxenmq/CMakeLists.txt new file mode 100644 index 0000000..efd2e32 --- /dev/null +++ b/pyoxenmq/CMakeLists.txt @@ -0,0 +1,10 @@ + +pybind11_add_module(pyoxenmq MODULE + bencode.cpp + oxenmq.cpp + module.cpp +) + +target_link_libraries(pyoxenmq PUBLIC PkgConfig::oxenmq) + + diff --git a/pylokimq/bencode.cpp b/pyoxenmq/bencode.cpp similarity index 76% rename from pylokimq/bencode.cpp rename to pyoxenmq/bencode.cpp index d039508..c63d9f7 100644 --- a/pylokimq/bencode.cpp +++ b/pyoxenmq/bencode.cpp @@ -1,7 +1,7 @@ #include "common.hpp" -#include "lokimq/base32z.h" +#include "oxenmq/base32z.h" -namespace lokimq +namespace oxenmq { void BEncode_Init(py::module & mod) @@ -10,7 +10,7 @@ namespace lokimq char * ptr = nullptr; py::ssize_t sz = 0; PyBytes_AsStringAndSize(data.ptr(), &ptr, &sz); - return lokimq::to_base32z(ptr, ptr+sz); + return oxenmq::to_base32z(ptr, ptr+sz); }); } } diff --git a/pylokimq/common.hpp b/pyoxenmq/common.hpp similarity index 78% rename from pylokimq/common.hpp rename to pyoxenmq/common.hpp index e92a459..a6b3d31 100644 --- a/pylokimq/common.hpp +++ b/pyoxenmq/common.hpp @@ -5,10 +5,10 @@ namespace py = pybind11; -namespace lokimq +namespace oxenmq { void - LokiMQ_Init(py::module &mod); + OxenMQ_Init(py::module &mod); void BEncode_Init(py::module & mod); diff --git a/pylokimq/lokimq.cpp b/pyoxenmq/lokimq.cpp similarity index 85% rename from pylokimq/lokimq.cpp rename to pyoxenmq/lokimq.cpp index 4c5fc5c..2b2e90e 100644 --- a/pylokimq/lokimq.cpp +++ b/pyoxenmq/lokimq.cpp @@ -1,18 +1,18 @@ #include "common.hpp" #include #include -#include -#include +#include +#include #include #include #include #include -namespace lokimq +namespace oxenmq { template - std::future> LokiMQ_start_request( - LokiMQ& lmq, + std::future> MQ_start_request( + OxenMQ& omq, ConnectionID conn, std::string name, std::vector byte_args, @@ -25,7 +25,7 @@ namespace lokimq auto result = std::make_shared>>(); auto fut = result->get_future(); - lmq.request(conn, std::move(name), + omq.request(conn, std::move(name), [result=std::move(result)](bool success, std::vector value) { if (success) @@ -70,7 +70,7 @@ namespace lokimq static std::mutex log_mutex; void - LokiMQ_Init(py::module & mod) + OxenMQ_Init(py::module & mod) { using namespace pybind11::literals; py::class_(mod, "ConnectionID") @@ -100,7 +100,7 @@ namespace lokimq return l; }); - py::class_(mod, "LokiMQ") + py::class_(mod, "OxenMQ") .def(py::init<>()) .def(py::init([](LogLevel level) { // Quick and dirty logger that logs to stderr. It would be much nicer to take a python @@ -110,42 +110,45 @@ namespace lokimq std::cerr << '[' << lvl << "][" << file << ':' << line << "]: " << msg << "\n"; }, level); })) - .def_readwrite("handshake_time", &LokiMQ::HANDSHAKE_TIME) - .def_readwrite("pubkey_base_routing_id", &LokiMQ::PUBKEY_BASED_ROUTING_ID) - .def_readwrite("max_message_size", &LokiMQ::MAX_MSG_SIZE) - .def_readwrite("max_sockets", &LokiMQ::MAX_SOCKETS) - .def_readwrite("reconnect_interval", &LokiMQ::RECONNECT_INTERVAL) - .def_readwrite("close_longer", &LokiMQ::CLOSE_LINGER) - .def_readwrite("connection_check_interval", &LokiMQ::CONN_CHECK_INTERVAL) - .def_readwrite("connection_heartbeat", &LokiMQ::CONN_HEARTBEAT) - .def_readwrite("connection_heartbeat_timeout", &LokiMQ::CONN_HEARTBEAT_TIMEOUT) - .def_readwrite("startup_umask", &LokiMQ::STARTUP_UMASK) - .def("start", &LokiMQ::start) + .def_readwrite("handshake_time", &OxenMQ::HANDSHAKE_TIME) + .def_readwrite("pubkey_base_routing_id", &OxenMQ::PUBKEY_BASED_ROUTING_ID) + .def_readwrite("max_message_size", &OxenMQ::MAX_MSG_SIZE) + .def_readwrite("max_sockets", &OxenMQ::MAX_SOCKETS) + .def_readwrite("reconnect_interval", &OxenMQ::RECONNECT_INTERVAL) + .def_readwrite("close_longer", &OxenMQ::CLOSE_LINGER) + .def_readwrite("connection_check_interval", &OxenMQ::CONN_CHECK_INTERVAL) + .def_readwrite("connection_heartbeat", &OxenMQ::CONN_HEARTBEAT) + .def_readwrite("connection_heartbeat_timeout", &OxenMQ::CONN_HEARTBEAT_TIMEOUT) + .def_readwrite("startup_umask", &OxenMQ::STARTUP_UMASK) + .def("start", &OxenMQ::start) .def("listen_plain", - [](LokiMQ & self, std::string path) { + [](OxenMQ & self, std::string path) { self.listen_plain(path); }) - .def("listen_curve", &LokiMQ::listen_curve) + .def("listen_curve", + [](OxenMQ & self, std::string path) { + self.listen_curve(path); + }) .def("add_tagged_thread", - [](LokiMQ & self, std::string name) { + [](OxenMQ & self, std::string name) { return self.add_tagged_thread(name); }) .def("add_timer", - [](LokiMQ & self, std::chrono::milliseconds interval, std::function callback) { + [](OxenMQ & self, std::chrono::milliseconds interval, std::function callback) { self.add_timer(callback, interval); }) .def("call_soon", - [](LokiMQ & self, std::function job, std::optional thread) + [](OxenMQ & self, std::function job, std::optional thread) { self.job(std::move(job), std::move(thread)); }) .def("add_anonymous_category", - [](LokiMQ & self, std::string name) + [](OxenMQ & self, std::string name) { self.add_category(std::move(name), AuthLevel::none); }) .def("add_request_command", - [](LokiMQ &self, + [](OxenMQ &self, std::string category, std::string name, py::function handler) @@ -176,7 +179,7 @@ namespace lokimq }); }) .def("add_request_command_ex", - [](LokiMQ &self, + [](OxenMQ &self, std::string category, std::string name, py::function handler) @@ -207,7 +210,7 @@ namespace lokimq }); }) .def("connect_remote", - [](LokiMQ & self, + [](OxenMQ & self, std::string remote) -> ConnectionID { std::promise promise; @@ -221,7 +224,7 @@ namespace lokimq return promise.get_future().get(); }) .def("request", - [](LokiMQ & self, + [](OxenMQ & self, ConnectionID conn, std::string name, std::vector args, @@ -236,7 +239,7 @@ namespace lokimq }, "conn"_a, "name"_a, "args"_a = std::vector{}, "timeout"_a = py::none{}) .def("request_future", - [](LokiMQ & self, + [](OxenMQ & self, ConnectionID conn, std::string name, std::vector args, diff --git a/pyoxenmq/module.cpp b/pyoxenmq/module.cpp new file mode 100644 index 0000000..24842f6 --- /dev/null +++ b/pyoxenmq/module.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +PYBIND11_MODULE(pyoxenmq, m) +{ + oxenmq::OxenMQ_Init(m); + oxenmq::BEncode_Init(m); +} diff --git a/pyoxenmq/oxenmq.cpp b/pyoxenmq/oxenmq.cpp new file mode 100644 index 0000000..d70d1e4 --- /dev/null +++ b/pyoxenmq/oxenmq.cpp @@ -0,0 +1,255 @@ +#include "common.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace oxenmq +{ + template + std::future> MQ_start_request( + OxenMQ& omq, + ConnectionID conn, + std::string name, + std::vector byte_args, + Options&&... opts) + { + std::vector args; + args.reserve(byte_args.size()); + for (auto& b : byte_args) + args.push_back(b); + + auto result = std::make_shared>>(); + auto fut = result->get_future(); + omq.request(conn, std::move(name), + [result=std::move(result)](bool success, std::vector value) + { + if (success) + result->set_value(std::move(value)); + else + { + std::string err; + for (auto& m : value) { + if (!err.empty()) err += ", "; + err += m; + } + result->set_exception(std::make_exception_ptr(std::runtime_error{"Request failed: " + err})); + } + }, + oxenmq::send_option::data_parts(args.begin(), args.end()), + std::forward(opts)... + ); + return fut; + } + + // Binds a stl future. `Conv` is a lambda that converts the future's .get() value into something + // Python-y (it can be the value directly, if the value is convertible to Python already). + template + void bind_future(py::module& m, std::string class_name, Conv conv) + { + py::class_(m, class_name.c_str()) + .def("get", [conv=std::move(conv)](F& f) { return conv(f.get()); }, + "Gets the result (or raises an exception if the result set an exception); must only be called once") + .def("valid", [](F& f) { return f.valid(); }, + "Returns true if the result is available") + .def("wait", &F::wait, + "Waits indefinitely for the result to become available") + .def("wait_for", &F::template wait_for>, + "Waits up to the given timedelta for the result to become available") + .def("wait_for", [](F& f, double seconds) { return f.wait_for(std::chrono::duration{seconds}); }, + "Waits up to the given number of seconds for the result to become available") + .def("wait_until", &F::template wait_until, + "Wait until the given datetime for the result to become available") + ; + } + + static std::mutex log_mutex; + + void + OxenMQ_Init(py::module & mod) + { + using namespace pybind11::literals; + py::class_(mod, "ConnectionID") + .def("__eq__", [](const ConnectionID & self, const ConnectionID & other) { + return self == other; + }); + py::class_(mod, "Message") + .def_readonly("remote", &Message::remote) + .def_readonly("conn", &Message::conn); + + py::class_
(mod, "Address") + .def(py::init()); + py::class_(mod, "TaggedThreadID"); + py::enum_(mod, "LogLevel") + .value("fatal", LogLevel::fatal).value("error", LogLevel::error).value("warn", LogLevel::warn) + .value("info", LogLevel::info).value("debug", LogLevel::debug).value("trace", LogLevel::trace); + + py::enum_(mod, "future_status") + .value("deferred", std::future_status::deferred) + .value("ready", std::future_status::ready) + .value("timeout", std::future_status::timeout); + bind_future>>(mod, "ResultFuture", + [](std::vector bytes) { + py::list l; + for (const auto& v : bytes) + l.append(py::bytes(v)); + return l; + }); + + py::class_(mod, "OxenMQ") + .def(py::init<>()) + .def(py::init([](LogLevel level) { + // Quick and dirty logger that logs to stderr. It would be much nicer to take a python + // function, but that deadlocks pretty much right away because of the crappiness of the gil. + return std::make_unique([] (LogLevel lvl, const char* file, int line, std::string msg) mutable { + std::lock_guard l{log_mutex}; + std::cerr << '[' << lvl << "][" << file << ':' << line << "]: " << msg << "\n"; + }, level); + })) + .def_readwrite("handshake_time", &OxenMQ::HANDSHAKE_TIME) + .def_readwrite("pubkey_base_routing_id", &OxenMQ::PUBKEY_BASED_ROUTING_ID) + .def_readwrite("max_message_size", &OxenMQ::MAX_MSG_SIZE) + .def_readwrite("max_sockets", &OxenMQ::MAX_SOCKETS) + .def_readwrite("reconnect_interval", &OxenMQ::RECONNECT_INTERVAL) + .def_readwrite("close_longer", &OxenMQ::CLOSE_LINGER) + .def_readwrite("connection_check_interval", &OxenMQ::CONN_CHECK_INTERVAL) + .def_readwrite("connection_heartbeat", &OxenMQ::CONN_HEARTBEAT) + .def_readwrite("connection_heartbeat_timeout", &OxenMQ::CONN_HEARTBEAT_TIMEOUT) + .def_readwrite("startup_umask", &OxenMQ::STARTUP_UMASK) + .def("start", &OxenMQ::start) + .def("listen_plain", + [](OxenMQ & self, std::string path) { + self.listen_plain(path); + }) + .def("listen_curve", + [](OxenMQ & self, std::string path) { + self.listen_curve(path); + }) + .def("add_tagged_thread", + [](OxenMQ & self, std::string name) { + return self.add_tagged_thread(name); + }) + .def("add_timer", + [](OxenMQ & self, std::chrono::milliseconds interval, std::function callback) { + self.add_timer(callback, interval); + }) + .def("call_soon", + [](OxenMQ & self, std::function job, std::optional thread) + { + self.job(std::move(job), std::move(thread)); + }) + .def("add_anonymous_category", + [](OxenMQ & self, std::string name) + { + self.add_category(std::move(name), AuthLevel::none); + }) + .def("add_request_command", + [](OxenMQ &self, + std::string category, + std::string name, + py::function handler) + { + self.add_request_command(category, name, + [handler](Message & msg) { + std::string result; + { + py::gil_scoped_acquire gil; + + std::vector data; + for (auto& arg : msg.data) + { + data.emplace_back(arg.begin(), arg.size()); + } + + try + { + const auto obj = handler(data); + result = py::str(obj); + } + catch(std::exception & ex) + { + PyErr_SetString(PyExc_RuntimeError, ex.what()); + } + } + msg.send_reply(result); + }); + }) + .def("add_request_command_ex", + [](OxenMQ &self, + std::string category, + std::string name, + py::function handler) + { + self.add_request_command(category, name, + [handler](Message & msg) { + std::string result; + { + py::gil_scoped_acquire gil; + + std::vector data; + for (auto& arg : msg.data) + { + data.emplace_back(arg.begin(), arg.size()); + } + + try + { + const auto obj = handler(data, msg); + result = py::str(obj); + } + catch(std::exception & ex) + { + PyErr_SetString(PyExc_RuntimeError, ex.what()); + } + } + msg.send_reply(result); + }); + }) + .def("connect_remote", + [](OxenMQ & self, + std::string remote) -> ConnectionID + { + std::promise promise; + self.connect_remote( + remote, + [&promise](ConnectionID id) { promise.set_value(std::move(id)); }, + [&promise](auto, std::string_view reason) { + promise.set_exception(std::make_exception_ptr( + std::runtime_error{"Connection failed: " + std::string{reason}})); + }); + return promise.get_future().get(); + }) + .def("request", + [](OxenMQ & self, + ConnectionID conn, + std::string name, + std::vector args, + std::optional timeout) -> py::list + { + py::list l; + for (auto& s : MQ_start_request(self, conn, std::move(name), std::move(args), + oxenmq::send_option::request_timeout{timeout ? std::chrono::milliseconds(long(*timeout * 1000)) : DEFAULT_REQUEST_TIMEOUT} + ).get()) + l.append(py::bytes(s)); + return l; + }, + "conn"_a, "name"_a, "args"_a = std::vector{}, "timeout"_a = py::none{}) + .def("request_future", + [](OxenMQ & self, + ConnectionID conn, + std::string name, + std::vector args, + std::optional timeout) -> std::future> + { + return MQ_start_request(self, conn, std::move(name), std::move(args), + oxenmq::send_option::request_timeout{timeout ? std::chrono::milliseconds(long(*timeout * 1000)) : DEFAULT_REQUEST_TIMEOUT} + ); + }, + "conn"_a, "name"_a, "args"_a = std::vector{}, "timeout"_a = py::none{}) + ; + } +} diff --git a/setup.py b/setup.py index 9bbd54c..ef334fb 100644 --- a/setup.py +++ b/setup.py @@ -61,13 +61,13 @@ class CMakeBuild(build_ext): subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp) setup( - name='pylokimq', - version='0.0.1', + name='pyoxenmq', + version='0.1.0', author='Jeff Becker', author_email='jeff@i2p.rocks', - description='pybind lokimq bindings', + description='pybind oxenmq bindings', long_description='', - ext_modules=[CMakeExtension('pylokimq')], + ext_modules=[CMakeExtension('pyoxenmq')], packages=["lokinet.auth"], cmdclass=dict(build_ext=CMakeBuild), zip_safe=False,