oxen-io
/
oxen-pyoxenmq
mirror of https://github.com/oxen-io/oxen-pyoxenmq.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

loki -> oxen refactor 

* rename all namespaces so it compiles again
This commit is contained in:
Jeff Becker 2021-02-01 10:48:27 -05:00
parent aade6d835c
commit 4597f5ca6c
No known key found for this signature in database
GPG Key ID: F357B3B42F6F9B05
15 changed files with 342 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitmodules vendored
View File

@ -2,7 +2,7 @@
path = external/pybind11
url = https://github.com/pybind/pybind11
branch = stable
[submodule "external/loki-mq"]
path = external/loki-mq
url = https://github.com/loki-project/loki-mq
[submodule "external/oxen-mq"]
path = external/oxen-mq
url = https://github.com/oxen-io/loki-mq
branch = dev

12
CMakeLists.txt
View File

@ -8,10 +8,10 @@ if(CCACHE_PROGRAM)
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}")
endif()
set(PROJECT_NAME pylokimq)
set(PROJECT_NAME pyoxenmq)
project(${PROJECT_NAME}
VERSION 0.0.1
DESCRIPTION "pybind layer for lokimq"
VERSION 0.1.0
DESCRIPTION "pybind layer for oxenmq"
LANGUAGES CXX)
@ -41,7 +41,7 @@ if(SUBMODULE_CHECK)
endfunction ()
check_submodule(external/pybind11)
check_submodule(external/loki-mq)
check_submodule(external/oxen-mq)
endif()
endif()
@ -50,6 +50,6 @@ add_compile_options(-Wno-deprecated-declarations)
add_subdirectory(external/pybind11)
include(FindPkgConfig)
pkg_check_modules(lokimq REQUIRED IMPORTED_TARGET GLOBAL liblokimq)
pkg_check_modules(oxenmq REQUIRED IMPORTED_TARGET GLOBAL liboxenmq)
add_subdirectory(pylokimq)
add_subdirectory(pyoxenmq)

16
examples/client.py
View File

@ -1,20 +1,20 @@
import pylokimq
import pyoxenmq
def do_connected(lmq, conn):
def do_connected(mq, conn):
print("connected via", conn)
return lmq.request(conn, "llarp.auth", ["dq3j4dj99w6wi4t4yjnya8sxtqr1rojt8jgnn6467o6aoenm3o3o.loki", "5:token"])
return mq.request(conn, "llarp.auth", ["dq3j4dj99w6wi4t4yjnya8sxtqr1rojt8jgnn6467o6aoenm3o3o.loki", "5:token"])
def do_request(lmq):
def do_request(mq):
print('connect')
conn = lmq.connect_remote("ipc:///tmp/lmq.sock")
conn = mq.connect_remote("ipc:///tmp/lmq.sock")
if conn:
return do_connected(lmq, conn)
def main():
lmq = pylokimq.LokiMQ()
mq = pyoxenmq.OxenMQ()
print("start")
lmq.start()
print(do_request(lmq))
mq.start()
print(do_request(mq))
print("done")
if __name__ == '__main__':

4
examples/server.py
View File

@ -1,4 +1,4 @@
import pylokimq
import pyoxenmq
import time
def handle_auth(args):
@ -7,7 +7,7 @@ def handle_auth(args):
return "OK"
def main():
lmq = pylokimq.LokiMQ()
lmq = pyoxenmq.OxenMQ()
lmq.listen_plain("ipc:///tmp/lmq.sock")
lmq.add_anonymous_category("llarp")
lmq.add_request_command("llarp", "auth", handle_auth)

4
examples/test.py
View File

@ -1,11 +1,11 @@
import pylokimq
import pyoxenmq
import time
def handle_ping(args):
print(args)
return args
lmq = pylokimq.LokiMQ()
lmq = pyoxenmq.OxenMQ()
lmq.listen_plain("ipc:///tmp/lmq.sock")
lmq.add_anonymous_category("python")
lmq.add_request_command("python", "ping", handle_ping)

14
lokinet/auth/__main__.py
View File

@ -1,4 +1,4 @@
import pylokimq
import pyoxenmq
import base64
import subprocess
import shlex
@ -61,7 +61,7 @@ def decode_value(data, first=None):
def decode_address(data):
return '{}.loki'.format(pylokimq.base32z_encode(decode_value(data)[b's'][b's']))
return '{}.loki'.format(pyoxenmq.base32z_encode(decode_value(data)[b's'][b's']))
def handle_auth_impl(args, cmd):
cmd2 = cmd
@ -85,11 +85,11 @@ def main():
ap.add_argument("--cmd", required=True, help="script to call for authentication")
args = ap.parse_args()
cmd = shlex.split(args.cmd)
lmq = pylokimq.LokiMQ()
lmq.listen_plain(args.bind)
lmq.add_anonymous_category("llarp")
lmq.add_request_command("llarp", "auth", lambda x : handle_auth(x, cmd))
lmq.start()
mq = pyoxenmq.OxenMQ()
mq.listen_plain(args.bind)
mq.add_anonymous_category("llarp")
mq.add_request_command("llarp", "auth", lambda x : handle_auth(x, cmd))
mq.start()
print("server started")
while True:
time.sleep(1)

10
pylokimq/CMakeLists.txt
View File

@ -1,10 +0,0 @@
pybind11_add_module(pylokimq MODULE
bencode.cpp
lokimq.cpp
module.cpp
)
target_link_libraries(pylokimq PUBLIC PkgConfig::lokimq)

7
pylokimq/module.cpp
View File

@ -1,7 +0,0 @@
#include "common.hpp"
PYBIND11_MODULE(pylokimq, m)
{
lokimq::LokiMQ_Init(m);
lokimq::BEncode_Init(m);
}

10
pyoxenmq/CMakeLists.txt Normal file
View File

@ -0,0 +1,10 @@
pybind11_add_module(pyoxenmq MODULE
bencode.cpp
oxenmq.cpp
module.cpp
)
target_link_libraries(pyoxenmq PUBLIC PkgConfig::oxenmq)

6
pylokimq/bencode.cpp → pyoxenmq/bencode.cpp
View File

@ -1,7 +1,7 @@
#include "common.hpp"
#include "lokimq/base32z.h"
#include "oxenmq/base32z.h"
namespace lokimq
namespace oxenmq
{
void
BEncode_Init(py::module & mod)
@ -10,7 +10,7 @@ namespace lokimq
char * ptr = nullptr;
py::ssize_t sz = 0;
PyBytes_AsStringAndSize(data.ptr(), &ptr, &sz);
return lokimq::to_base32z(ptr, ptr+sz);
return oxenmq::to_base32z(ptr, ptr+sz);
});
}
}

4
pylokimq/common.hpp → pyoxenmq/common.hpp
View File

@ -5,10 +5,10 @@
namespace py = pybind11;
namespace lokimq
namespace oxenmq
{
void
LokiMQ_Init(py::module &mod);
OxenMQ_Init(py::module &mod);
void
BEncode_Init(py::module & mod);

63
pylokimq/lokimq.cpp → pyoxenmq/lokimq.cpp
View File

@ -1,18 +1,18 @@
#include "common.hpp"
#include <chrono>
#include <exception>
#include <lokimq/lokimq.h>
#include <lokimq/address.h>
#include <oxenmq/oxenmq.h>
#include <oxenmq/address.h>
#include <pybind11/chrono.h>
#include <pybind11/functional.h>
#include <future>
#include <memory>
namespace lokimq
namespace oxenmq
{
template <typename... Options>
std::future<std::vector<std::string>> LokiMQ_start_request(
LokiMQ& lmq,
std::future<std::vector<std::string>> MQ_start_request(
OxenMQ& omq,
ConnectionID conn,
std::string name,
std::vector<py::bytes> byte_args,
@ -25,7 +25,7 @@ namespace lokimq
auto result = std::make_shared<std::promise<std::vector<std::string>>>();
auto fut = result->get_future();
lmq.request(conn, std::move(name),
omq.request(conn, std::move(name),
[result=std::move(result)](bool success, std::vector<std::string> value)
{
if (success)
@ -70,7 +70,7 @@ namespace lokimq
static std::mutex log_mutex;
void
LokiMQ_Init(py::module & mod)
OxenMQ_Init(py::module & mod)
{
using namespace pybind11::literals;
py::class_<ConnectionID>(mod, "ConnectionID")
@ -100,7 +100,7 @@ namespace lokimq
return l;
});
py::class_<LokiMQ>(mod, "LokiMQ")
py::class_<OxenMQ>(mod, "OxenMQ")
.def(py::init<>())
.def(py::init([](LogLevel level) {
// Quick and dirty logger that logs to stderr. It would be much nicer to take a python
@ -110,42 +110,45 @@ namespace lokimq
std::cerr << '[' << lvl << "][" << file << ':' << line << "]: " << msg << "\n";
}, level);
}))
.def_readwrite("handshake_time", &LokiMQ::HANDSHAKE_TIME)
.def_readwrite("pubkey_base_routing_id", &LokiMQ::PUBKEY_BASED_ROUTING_ID)
.def_readwrite("max_message_size", &LokiMQ::MAX_MSG_SIZE)
.def_readwrite("max_sockets", &LokiMQ::MAX_SOCKETS)
.def_readwrite("reconnect_interval", &LokiMQ::RECONNECT_INTERVAL)
.def_readwrite("close_longer", &LokiMQ::CLOSE_LINGER)
.def_readwrite("connection_check_interval", &LokiMQ::CONN_CHECK_INTERVAL)
.def_readwrite("connection_heartbeat", &LokiMQ::CONN_HEARTBEAT)
.def_readwrite("connection_heartbeat_timeout", &LokiMQ::CONN_HEARTBEAT_TIMEOUT)
.def_readwrite("startup_umask", &LokiMQ::STARTUP_UMASK)
.def("start", &LokiMQ::start)
.def_readwrite("handshake_time", &OxenMQ::HANDSHAKE_TIME)
.def_readwrite("pubkey_base_routing_id", &OxenMQ::PUBKEY_BASED_ROUTING_ID)
.def_readwrite("max_message_size", &OxenMQ::MAX_MSG_SIZE)
.def_readwrite("max_sockets", &OxenMQ::MAX_SOCKETS)
.def_readwrite("reconnect_interval", &OxenMQ::RECONNECT_INTERVAL)
.def_readwrite("close_longer", &OxenMQ::CLOSE_LINGER)
.def_readwrite("connection_check_interval", &OxenMQ::CONN_CHECK_INTERVAL)
.def_readwrite("connection_heartbeat", &OxenMQ::CONN_HEARTBEAT)
.def_readwrite("connection_heartbeat_timeout", &OxenMQ::CONN_HEARTBEAT_TIMEOUT)
.def_readwrite("startup_umask", &OxenMQ::STARTUP_UMASK)
.def("start", &OxenMQ::start)
.def("listen_plain",
[](LokiMQ & self, std::string path) {
[](OxenMQ & self, std::string path) {
self.listen_plain(path);
})
.def("listen_curve", &LokiMQ::listen_curve)
.def("listen_curve",
[](OxenMQ & self, std::string path) {
self.listen_curve(path);
})
.def("add_tagged_thread",
[](LokiMQ & self, std::string name) {
[](OxenMQ & self, std::string name) {
return self.add_tagged_thread(name);
})
.def("add_timer",
[](LokiMQ & self, std::chrono::milliseconds interval, std::function<void(void)> callback) {
[](OxenMQ & self, std::chrono::milliseconds interval, std::function<void(void)> callback) {
self.add_timer(callback, interval);
})
.def("call_soon",
[](LokiMQ & self, std::function<void(void)> job, std::optional<TaggedThreadID> thread)
[](OxenMQ & self, std::function<void(void)> job, std::optional<TaggedThreadID> thread)
{
self.job(std::move(job), std::move(thread));
})
.def("add_anonymous_category",
[](LokiMQ & self, std::string name)
[](OxenMQ & self, std::string name)
{
self.add_category(std::move(name), AuthLevel::none);
})
.def("add_request_command",
[](LokiMQ &self,
[](OxenMQ &self,
std::string category,
std::string name,
py::function handler)
@ -176,7 +179,7 @@ namespace lokimq
});
})
.def("add_request_command_ex",
[](LokiMQ &self,
[](OxenMQ &self,
std::string category,
std::string name,
py::function handler)
@ -207,7 +210,7 @@ namespace lokimq
});
})
.def("connect_remote",
[](LokiMQ & self,
[](OxenMQ & self,
std::string remote) -> ConnectionID
{
std::promise<ConnectionID> promise;
@ -221,7 +224,7 @@ namespace lokimq
return promise.get_future().get();
})
.def("request",
[](LokiMQ & self,
[](OxenMQ & self,
ConnectionID conn,
std::string name,
std::vector<py::bytes> args,
@ -236,7 +239,7 @@ namespace lokimq
},
"conn"_a, "name"_a, "args"_a = std::vector<py::bytes>{}, "timeout"_a = py::none{})
.def("request_future",
[](LokiMQ & self,
[](OxenMQ & self,
ConnectionID conn,
std::string name,
std::vector<py::bytes> args,

7
pyoxenmq/module.cpp Normal file
View File

@ -0,0 +1,7 @@
#include "common.hpp"
PYBIND11_MODULE(pyoxenmq, m)
{
oxenmq::OxenMQ_Init(m);
oxenmq::BEncode_Init(m);
}

255
pyoxenmq/oxenmq.cpp Normal file
View File

@ -0,0 +1,255 @@
#include "common.hpp"
#include <chrono>
#include <exception>
#include <oxenmq/oxenmq.h>
#include <oxenmq/address.h>
#include <pybind11/chrono.h>
#include <pybind11/functional.h>
#include <future>
#include <memory>
namespace oxenmq
{
template <typename... Options>
std::future<std::vector<std::string>> MQ_start_request(
OxenMQ& omq,
ConnectionID conn,
std::string name,
std::vector<py::bytes> byte_args,
Options&&... opts)
{
std::vector<std::string> args;
args.reserve(byte_args.size());
for (auto& b : byte_args)
args.push_back(b);
auto result = std::make_shared<std::promise<std::vector<std::string>>>();
auto fut = result->get_future();
omq.request(conn, std::move(name),
[result=std::move(result)](bool success, std::vector<std::string> value)
{
if (success)
result->set_value(std::move(value));
else
{
std::string err;
for (auto& m : value) {
if (!err.empty()) err += ", ";
err += m;
}
result->set_exception(std::make_exception_ptr(std::runtime_error{"Request failed: " + err}));
}
},
oxenmq::send_option::data_parts(args.begin(), args.end()),
std::forward<Options>(opts)...
);
return fut;
}
// Binds a stl future. `Conv` is a lambda that converts the future's .get() value into something
// Python-y (it can be the value directly, if the value is convertible to Python already).
template <typename F, typename Conv>
void bind_future(py::module& m, std::string class_name, Conv conv)
{
py::class_<F>(m, class_name.c_str())
.def("get", [conv=std::move(conv)](F& f) { return conv(f.get()); },
"Gets the result (or raises an exception if the result set an exception); must only be called once")
.def("valid", [](F& f) { return f.valid(); },
"Returns true if the result is available")
.def("wait", &F::wait,
"Waits indefinitely for the result to become available")
.def("wait_for", &F::template wait_for<double, std::ratio<1>>,
"Waits up to the given timedelta for the result to become available")
.def("wait_for", [](F& f, double seconds) { return f.wait_for(std::chrono::duration<double>{seconds}); },
"Waits up to the given number of seconds for the result to become available")
.def("wait_until", &F::template wait_until<std::chrono::system_clock, std::chrono::system_clock::duration>,
"Wait until the given datetime for the result to become available")
;
}
static std::mutex log_mutex;
void
OxenMQ_Init(py::module & mod)
{
using namespace pybind11::literals;
py::class_<ConnectionID>(mod, "ConnectionID")
.def("__eq__", [](const ConnectionID & self, const ConnectionID & other) {
return self == other;
});
py::class_<Message>(mod, "Message")
.def_readonly("remote", &Message::remote)
.def_readonly("conn", &Message::conn);
py::class_<address>(mod, "Address")
.def(py::init<std::string>());
py::class_<TaggedThreadID>(mod, "TaggedThreadID");
py::enum_<LogLevel>(mod, "LogLevel")
.value("fatal", LogLevel::fatal).value("error", LogLevel::error).value("warn", LogLevel::warn)
.value("info", LogLevel::info).value("debug", LogLevel::debug).value("trace", LogLevel::trace);
py::enum_<std::future_status>(mod, "future_status")
.value("deferred", std::future_status::deferred)
.value("ready", std::future_status::ready)
.value("timeout", std::future_status::timeout);
bind_future<std::future<std::vector<std::string>>>(mod, "ResultFuture",
[](std::vector<std::string> bytes) {
py::list l;
for (const auto& v : bytes)
l.append(py::bytes(v));
return l;
});
py::class_<OxenMQ>(mod, "OxenMQ")
.def(py::init<>())
.def(py::init([](LogLevel level) {
// Quick and dirty logger that logs to stderr. It would be much nicer to take a python
// function, but that deadlocks pretty much right away because of the crappiness of the gil.
return std::make_unique<OxenMQ>([] (LogLevel lvl, const char* file, int line, std::string msg) mutable {
std::lock_guard l{log_mutex};
std::cerr << '[' << lvl << "][" << file << ':' << line << "]: " << msg << "\n";
}, level);
}))
.def_readwrite("handshake_time", &OxenMQ::HANDSHAKE_TIME)
.def_readwrite("pubkey_base_routing_id", &OxenMQ::PUBKEY_BASED_ROUTING_ID)
.def_readwrite("max_message_size", &OxenMQ::MAX_MSG_SIZE)
.def_readwrite("max_sockets", &OxenMQ::MAX_SOCKETS)
.def_readwrite("reconnect_interval", &OxenMQ::RECONNECT_INTERVAL)
.def_readwrite("close_longer", &OxenMQ::CLOSE_LINGER)
.def_readwrite("connection_check_interval", &OxenMQ::CONN_CHECK_INTERVAL)
.def_readwrite("connection_heartbeat", &OxenMQ::CONN_HEARTBEAT)
.def_readwrite("connection_heartbeat_timeout", &OxenMQ::CONN_HEARTBEAT_TIMEOUT)
.def_readwrite("startup_umask", &OxenMQ::STARTUP_UMASK)
.def("start", &OxenMQ::start)
.def("listen_plain",
[](OxenMQ & self, std::string path) {
self.listen_plain(path);
})
.def("listen_curve",
[](OxenMQ & self, std::string path) {
self.listen_curve(path);
})
.def("add_tagged_thread",
[](OxenMQ & self, std::string name) {
return self.add_tagged_thread(name);
})
.def("add_timer",
[](OxenMQ & self, std::chrono::milliseconds interval, std::function<void(void)> callback) {
self.add_timer(callback, interval);
})
.def("call_soon",
[](OxenMQ & self, std::function<void(void)> job, std::optional<TaggedThreadID> thread)
{
self.job(std::move(job), std::move(thread));
})
.def("add_anonymous_category",
[](OxenMQ & self, std::string name)
{
self.add_category(std::move(name), AuthLevel::none);
})
.def("add_request_command",
[](OxenMQ &self,
std::string category,
std::string name,
py::function handler)
{
self.add_request_command(category, name,
[handler](Message & msg) {
std::string result;
{
py::gil_scoped_acquire gil;
std::vector<py::bytes> data;
for (auto& arg : msg.data)
{
data.emplace_back(arg.begin(), arg.size());
}
try
{
const auto obj = handler(data);
result = py::str(obj);
}
catch(std::exception & ex)
{
PyErr_SetString(PyExc_RuntimeError, ex.what());
}
}
msg.send_reply(result);
});
})
.def("add_request_command_ex",
[](OxenMQ &self,
std::string category,
std::string name,
py::function handler)
{
self.add_request_command(category, name,
[handler](Message & msg) {
std::string result;
{
py::gil_scoped_acquire gil;
std::vector<py::bytes> data;
for (auto& arg : msg.data)
{
data.emplace_back(arg.begin(), arg.size());
}
try
{
const auto obj = handler(data, msg);
result = py::str(obj);
}
catch(std::exception & ex)
{
PyErr_SetString(PyExc_RuntimeError, ex.what());
}
}
msg.send_reply(result);
});
})
.def("connect_remote",
[](OxenMQ & self,
std::string remote) -> ConnectionID
{
std::promise<ConnectionID> promise;
self.connect_remote(
remote,
[&promise](ConnectionID id) { promise.set_value(std::move(id)); },
[&promise](auto, std::string_view reason) {
promise.set_exception(std::make_exception_ptr(
std::runtime_error{"Connection failed: " + std::string{reason}}));
});
return promise.get_future().get();
})
.def("request",
[](OxenMQ & self,
ConnectionID conn,
std::string name,
std::vector<py::bytes> args,
std::optional<double> timeout) -> py::list
{
py::list l;
for (auto& s : MQ_start_request(self, conn, std::move(name), std::move(args),
oxenmq::send_option::request_timeout{timeout ? std::chrono::milliseconds(long(*timeout * 1000)) : DEFAULT_REQUEST_TIMEOUT}
).get())
l.append(py::bytes(s));
return l;
},
"conn"_a, "name"_a, "args"_a = std::vector<py::bytes>{}, "timeout"_a = py::none{})
.def("request_future",
[](OxenMQ & self,
ConnectionID conn,
std::string name,
std::vector<py::bytes> args,
std::optional<double> timeout) -> std::future<std::vector<std::string>>
{
return MQ_start_request(self, conn, std::move(name), std::move(args),
oxenmq::send_option::request_timeout{timeout ? std::chrono::milliseconds(long(*timeout * 1000)) : DEFAULT_REQUEST_TIMEOUT}
);
},
"conn"_a, "name"_a, "args"_a = std::vector<py::bytes>{}, "timeout"_a = py::none{})
;
}
}

8
setup.py
View File

@ -61,13 +61,13 @@ class CMakeBuild(build_ext):
subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp)
setup(
name='pylokimq',
version='0.0.1',
name='pyoxenmq',
version='0.1.0',
author='Jeff Becker',
author_email='jeff@i2p.rocks',
description='pybind lokimq bindings',
description='pybind oxenmq bindings',
long_description='',
ext_modules=[CMakeExtension('pylokimq')],
ext_modules=[CMakeExtension('pyoxenmq')],
packages=["lokinet.auth"],
cmdclass=dict(build_ext=CMakeBuild),
zip_safe=False,