Add v4 onion requests
Copied over from sogs (and slightly modified to remove optional sogs-specific bits). Adds a pytest suite with onion request tests. Also migrates to the deb.oxen.io oxenmq and pyonionreq python packages.
This commit is contained in:
parent
a20948c6b4
commit
c4e8767556
|
@ -1,6 +1,74 @@
|
||||||
local docker_base = 'registry.oxen.rocks/lokinet-ci-';
|
local docker_base = 'registry.oxen.rocks/lokinet-ci-';
|
||||||
local apt_get_quiet = 'apt-get -o=Dpkg::Use-Pty=0 -q';
|
local apt_get_quiet = 'apt-get -o=Dpkg::Use-Pty=0 -q';
|
||||||
|
|
||||||
|
local default_deps = [
|
||||||
|
'python3',
|
||||||
|
'python3-pytest',
|
||||||
|
'python3-coloredlogs',
|
||||||
|
'python3-flask',
|
||||||
|
'python3-pycryptodome',
|
||||||
|
'python3-nacl',
|
||||||
|
'python3-requests',
|
||||||
|
'python3-pyonionreq',
|
||||||
|
'python3-oxenmq',
|
||||||
|
];
|
||||||
|
|
||||||
|
// Regular build on a debian-like system:
|
||||||
|
local debian_pipeline(name,
|
||||||
|
image,
|
||||||
|
arch='amd64',
|
||||||
|
deps=default_deps,
|
||||||
|
before_pytest=[],
|
||||||
|
pytest_opts='',
|
||||||
|
extra_cmds=[],
|
||||||
|
services=[],
|
||||||
|
allow_fail=false) = {
|
||||||
|
kind: 'pipeline',
|
||||||
|
type: 'docker',
|
||||||
|
name: name,
|
||||||
|
platform: { arch: arch },
|
||||||
|
trigger: { branch: { exclude: ['debian/*', 'ubuntu/*'] } },
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
name: '🐍 pytest',
|
||||||
|
image: image,
|
||||||
|
pull: 'always',
|
||||||
|
[if allow_fail then 'failure']: 'ignore',
|
||||||
|
commands: [
|
||||||
|
'echo "Running on ${DRONE_STAGE_MACHINE}"',
|
||||||
|
'echo "man-db man-db/auto-update boolean false" | debconf-set-selections',
|
||||||
|
apt_get_quiet + ' update',
|
||||||
|
apt_get_quiet + ' install -y eatmydata',
|
||||||
|
'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y lsb-release',
|
||||||
|
'cp contrib/deb.oxen.io.gpg /etc/apt/trusted.gpg.d',
|
||||||
|
'echo deb http://deb.oxen.io $$(lsb_release -sc) main >/etc/apt/sources.list.d/oxen.list',
|
||||||
|
'eatmydata ' + apt_get_quiet + ' update',
|
||||||
|
'eatmydata ' + apt_get_quiet + ' dist-upgrade -y',
|
||||||
|
'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y ' + std.join(' ', deps),
|
||||||
|
'cp fileserver/config.py.sample fileserver/config.py',
|
||||||
|
] + before_pytest + [
|
||||||
|
'PYTHONPATH=. python3 -mpytest -vv --color=yes ' + pytest_opts,
|
||||||
|
]
|
||||||
|
+ extra_cmds,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
services: services,
|
||||||
|
};
|
||||||
|
|
||||||
|
local debian_pg_pipeline(name, image, pg_tag='bullseye') = debian_pipeline(
|
||||||
|
name,
|
||||||
|
image,
|
||||||
|
deps=default_deps + ['python3-pip', 'postgresql-client'],
|
||||||
|
services=[
|
||||||
|
{ name: 'pg', image: 'postgres:bullseye', environment: { POSTGRES_USER: 'ci', POSTGRES_PASSWORD: 'ci' } },
|
||||||
|
],
|
||||||
|
before_pytest=[
|
||||||
|
'pip3 install psycopg psycopg-pool',
|
||||||
|
'for i in $(seq 0 30); do if pg_isready -d ci -h pg -U ci -t 1; then break; fi; if [ "$i" = 30 ]; then echo "Timeout waiting for postgresql" >&2; exit 1; fi; sleep 1; done',
|
||||||
|
],
|
||||||
|
pytest_opts='--pgsql "postgresql://ci:ci@pg/ci"'
|
||||||
|
);
|
||||||
|
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
name: 'Lint checks',
|
name: 'Lint checks',
|
||||||
|
@ -28,4 +96,6 @@ local apt_get_quiet = 'apt-get -o=Dpkg::Use-Pty=0 -q';
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
|
||||||
|
debian_pg_pipeline('PostgreSQL 13/bullseye', docker_base + 'debian-stable', pg_tag='13-bullseye'),
|
||||||
]
|
]
|
||||||
|
|
1
.flake8
1
.flake8
|
@ -1,3 +1,4 @@
|
||||||
[flake8]
|
[flake8]
|
||||||
max-line-length = 100
|
max-line-length = 100
|
||||||
exclude=libonionrequests,fileserver/config.py
|
exclude=libonionrequests,fileserver/config.py
|
||||||
|
extend-ignore = E203 # See https://github.com/psf/black/issues/315
|
||||||
|
|
|
@ -1,3 +0,0 @@
|
||||||
[submodule "libonionrequests"]
|
|
||||||
path = libonionrequests
|
|
||||||
url = https://github.com/majestrate/libonionrequests.git
|
|
|
@ -21,10 +21,11 @@ Debian/Ubuntu).
|
||||||
- psycopg_pool
|
- psycopg_pool
|
||||||
- requests
|
- requests
|
||||||
- uwsgidecorators
|
- uwsgidecorators
|
||||||
|
- oxenmq
|
||||||
|
- pyonionreq
|
||||||
|
|
||||||
Additionally you need to build the Oxen Project's pyoxenmq and pylibonionreq. This repository links
|
(The last two are Oxen projects that either need to be installed manually, or via the Oxen deb
|
||||||
to them as submodules; `make` will build them locally for simple setups (proper deb packaging of
|
repository).
|
||||||
those libs is still a TODO).
|
|
||||||
|
|
||||||
### WSGI request handler
|
### WSGI request handler
|
||||||
|
|
||||||
|
|
Binary file not shown.
|
@ -1,5 +1,5 @@
|
||||||
from .web import app
|
from .web import app
|
||||||
from .db import psql
|
from . import db
|
||||||
from .timer import timer
|
from .timer import timer
|
||||||
from .stats import log_stats
|
from .stats import log_stats
|
||||||
|
|
||||||
|
@ -11,7 +11,7 @@ last_stats_printed = None
|
||||||
|
|
||||||
@timer(15)
|
@timer(15)
|
||||||
def periodic(signum):
|
def periodic(signum):
|
||||||
with app.app_context(), psql.cursor() as cur:
|
with app.app_context(), db.psql.cursor() as cur:
|
||||||
app.logger.debug("Cleaning up expired files")
|
app.logger.debug("Cleaning up expired files")
|
||||||
cur.execute("DELETE FROM files WHERE expiry <= NOW()")
|
cur.execute("DELETE FROM files WHERE expiry <= NOW()")
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
import nacl.public
|
import nacl.public
|
||||||
import os
|
import os
|
||||||
|
import pyonionreq.junk
|
||||||
|
|
||||||
from .web import app
|
from .web import app
|
||||||
|
|
||||||
|
@ -16,6 +17,9 @@ else:
|
||||||
with open("key_x25519", "wb") as f:
|
with open("key_x25519", "wb") as f:
|
||||||
f.write(privkey.encode())
|
f.write(privkey.encode())
|
||||||
|
|
||||||
|
_junk_parser = pyonionreq.junk.Parser(privkey=privkey.encode(), pubkey=privkey.public_key.encode())
|
||||||
|
parse_junk = _junk_parser.parse_junk
|
||||||
|
|
||||||
app.logger.info(
|
app.logger.info(
|
||||||
"File server pubkey: {}".format(
|
"File server pubkey: {}".format(
|
||||||
privkey.public_key.encode(encoder=nacl.encoding.HexEncoder).decode()
|
privkey.public_key.encode(encoder=nacl.encoding.HexEncoder).decode()
|
||||||
|
|
|
@ -10,8 +10,14 @@ from werkzeug.local import LocalProxy
|
||||||
@postfork
|
@postfork
|
||||||
def pg_connect():
|
def pg_connect():
|
||||||
global psql_pool
|
global psql_pool
|
||||||
|
|
||||||
|
# Test suite sets this to handle the connection itself:
|
||||||
|
if 'defer' in config.pgsql_connect_opts:
|
||||||
|
return
|
||||||
|
|
||||||
|
conninfo = config.pgsql_connect_opts.pop('conninfo', '')
|
||||||
psql_pool = ConnectionPool(
|
psql_pool = ConnectionPool(
|
||||||
min_size=2, max_size=32, kwargs={**config.pgsql_connect_opts, "autocommit": True}
|
conninfo, min_size=2, max_size=32, kwargs={**config.pgsql_connect_opts, "autocommit": True}
|
||||||
)
|
)
|
||||||
psql_pool.wait()
|
psql_pool.wait()
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
OK = 200
|
||||||
|
|
||||||
# error status codes:
|
# error status codes:
|
||||||
BAD_REQUEST = 400
|
BAD_REQUEST = 400
|
||||||
NOT_FOUND = 404
|
NOT_FOUND = 404
|
||||||
|
|
|
@ -1,131 +1,269 @@
|
||||||
from flask import request, Response
|
from flask import request, abort
|
||||||
import base64
|
|
||||||
import json
|
import json
|
||||||
from io import BytesIO
|
|
||||||
import pyonionreq.junk
|
|
||||||
|
|
||||||
from .web import app
|
from .web import app
|
||||||
from . import http
|
from . import crypto, http, utils
|
||||||
from . import crypto
|
from .subrequest import make_subrequest
|
||||||
|
|
||||||
import traceback
|
|
||||||
|
|
||||||
onionparser = pyonionreq.junk.Parser(
|
|
||||||
pubkey=crypto.privkey.public_key.encode(), privkey=crypto.privkey.encode()
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def handle_onionreq_plaintext(body):
|
def handle_v3_onionreq_plaintext(body):
|
||||||
"""
|
|
||||||
Handles a decrypted onion request; this injects a subrequest to process it then returns the
|
|
||||||
result of that subrequest (as bytes).
|
|
||||||
|
|
||||||
Note that this does not throw: if errors occur we map them into "success" responses with a body
|
|
||||||
of {"status_code":xxx} as onion requests have no ability at all to signal a request failure.
|
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
if body.startswith(b"{"):
|
if not body.startswith(b'{'):
|
||||||
# JSON input
|
raise RuntimeError("Invalid v3 onion request body: expected JSON object")
|
||||||
req = json.loads(body)
|
|
||||||
endpoint, method = req["endpoint"], req["method"]
|
|
||||||
subreq_headers = {k.lower(): v for k, v in req.get("headers", {}.items()).items()}
|
|
||||||
|
|
||||||
if method in http.BODY_METHODS:
|
req = json.loads(body)
|
||||||
if "body_binary" in req:
|
endpoint, method = req['endpoint'], req['method']
|
||||||
subreq_body = base64.b64decode(req["body_binary"], validate=True)
|
subreq_headers = {k.lower(): v for k, v in req.get('headers', {}).items()}
|
||||||
else:
|
|
||||||
subreq_body = req.get("body", "").encode()
|
|
||||||
ct = subreq_headers.pop(
|
|
||||||
"content-type",
|
|
||||||
"application/octet-stream" if "body_binary" in req else "application/json",
|
|
||||||
)
|
|
||||||
cl = len(subreq_body)
|
|
||||||
else:
|
|
||||||
subreq_body = b""
|
|
||||||
# Android bug workaround: Android Session (at least up to v1.11.12) sends a body on
|
|
||||||
# GET requests with a 4-character string "null" when it should send no body.
|
|
||||||
if "body" in req and len(req["body"]) == 4 and req["body"] == "null":
|
|
||||||
del req["body"]
|
|
||||||
|
|
||||||
if "body" in req and len(req["body"]) or "body_binary" in req:
|
|
||||||
raise RuntimeError(
|
|
||||||
"Invalid {} {} request: request must not contain a body".format(
|
|
||||||
method, endpoint
|
|
||||||
)
|
|
||||||
)
|
|
||||||
ct, cl = "", ""
|
|
||||||
|
|
||||||
for h in ("content-type", "content-length"):
|
|
||||||
if h in subreq_headers:
|
|
||||||
del subreq_headers[h]
|
|
||||||
|
|
||||||
elif body.startswith(b"d"):
|
|
||||||
raise RuntimeError("Bencoded onion requests not implemented yet")
|
|
||||||
|
|
||||||
|
if method in http.BODY_METHODS:
|
||||||
|
subreq_body = req.get('body', '').encode()
|
||||||
else:
|
else:
|
||||||
raise RuntimeError(
|
subreq_body = b''
|
||||||
"Invalid onion request body: expected JSON object or a bt-encoded dict"
|
# Android bug workaround: Android Session (at least up to v1.11.12) sends a body on
|
||||||
)
|
# GET requests with a 4-character string "null" when it should send no body.
|
||||||
|
if 'body' in req and len(req['body']) == 4 and req['body'] == 'null':
|
||||||
|
del req['body']
|
||||||
|
|
||||||
if "?" in endpoint:
|
if 'body' in req and req['body']:
|
||||||
endpoint, query_string = endpoint.split("?", 1)
|
raise RuntimeError(
|
||||||
else:
|
"Invalid {} {} request: request must not contain a body".format(
|
||||||
query_string = ""
|
method, endpoint
|
||||||
|
|
||||||
# Set up the wsgi environ variables for the subrequest (see PEP 0333)
|
|
||||||
subreq_env = {
|
|
||||||
**request.environ,
|
|
||||||
"REQUEST_METHOD": method,
|
|
||||||
"PATH_INFO": endpoint,
|
|
||||||
"QUERY_STRING": query_string,
|
|
||||||
"CONTENT_TYPE": ct,
|
|
||||||
"CONTENT_LENGTH": cl,
|
|
||||||
**{"HTTP_{}".format(h.upper().replace("-", "_")): v for h, v in subreq_headers.items()},
|
|
||||||
"wsgi.input": BytesIO(subreq_body),
|
|
||||||
}
|
|
||||||
|
|
||||||
try:
|
|
||||||
with app.request_context(subreq_env):
|
|
||||||
response = app.full_dispatch_request()
|
|
||||||
if response.status_code == 200:
|
|
||||||
data = response.get_data()
|
|
||||||
app.logger.debug(
|
|
||||||
"Onion sub-request for {} returned success, {} bytes".format(
|
|
||||||
endpoint, len(data)
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
return data
|
|
||||||
app.logger.warn(
|
if not endpoint.startswith('/'):
|
||||||
"Onion sub-request for {} {} returned status {}".format(
|
endpoint = '/' + endpoint
|
||||||
method, endpoint, response.status_code
|
|
||||||
)
|
response, _headers = make_subrequest(
|
||||||
|
method,
|
||||||
|
endpoint,
|
||||||
|
headers=subreq_headers,
|
||||||
|
body=subreq_body,
|
||||||
|
content_type='application/json',
|
||||||
|
)
|
||||||
|
|
||||||
|
if response.status_code == http.OK:
|
||||||
|
data = response.get_data()
|
||||||
|
app.logger.debug(
|
||||||
|
f"Onion sub-request for {endpoint} returned success, {len(data)} bytes"
|
||||||
)
|
)
|
||||||
return json.dumps({"status_code": response.status_code}).encode()
|
return data
|
||||||
except Exception:
|
return json.dumps({'status_code': response.status_code}).encode()
|
||||||
app.logger.warn(
|
|
||||||
"Onion sub-request for {} {} failed: {}".format(
|
|
||||||
method, endpoint, traceback.format_exc()
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return json.dumps({"status_code": http.BAD_GATEWAY}).encode()
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
app.logger.warn("Invalid onion request: {}".format(e))
|
app.logger.warning("Invalid onion request: {}".format(e))
|
||||||
return json.dumps({"status_code": http.BAD_REQUEST}).encode()
|
return json.dumps({'status_code': http.BAD_REQUEST}).encode()
|
||||||
|
|
||||||
|
|
||||||
|
def handle_v4_onionreq_plaintext(body):
|
||||||
|
try:
|
||||||
|
if not (body.startswith(b'l') and body.endswith(b'e')):
|
||||||
|
raise RuntimeError("Invalid onion request body: expected bencoded list")
|
||||||
|
|
||||||
|
belems = memoryview(body)[1:-1]
|
||||||
|
|
||||||
|
# Metadata json; this element is always required:
|
||||||
|
meta, belems = utils.bencode_consume_string(belems)
|
||||||
|
|
||||||
|
meta = json.loads(meta.tobytes())
|
||||||
|
|
||||||
|
# Then we can have a second optional string containing the body:
|
||||||
|
if len(belems) > 1:
|
||||||
|
subreq_body, belems = utils.bencode_consume_string(belems)
|
||||||
|
if len(belems):
|
||||||
|
raise RuntimeError("Invalid v4 onion request: found more than 2 parts")
|
||||||
|
else:
|
||||||
|
subreq_body = b''
|
||||||
|
|
||||||
|
method, endpoint = meta['method'], meta['endpoint']
|
||||||
|
if not endpoint.startswith('/'):
|
||||||
|
raise RuntimeError("Invalid v4 onion request: endpoint must start with /")
|
||||||
|
|
||||||
|
response, headers = make_subrequest(
|
||||||
|
method, endpoint, headers=meta.get('headers', {}), body=subreq_body
|
||||||
|
)
|
||||||
|
|
||||||
|
data = response.get_data()
|
||||||
|
app.logger.debug(
|
||||||
|
f"Onion sub-request for {endpoint} returned {response.status_code}, {len(data)} bytes"
|
||||||
|
)
|
||||||
|
|
||||||
|
meta = {'code': response.status_code, 'headers': headers}
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
app.logger.warning("Invalid v4 onion request: {}".format(e))
|
||||||
|
meta = {'code': http.BAD_REQUEST, 'headers': {'content-type': 'text/plain; charset=utf-8'}}
|
||||||
|
data = b'Invalid v4 onion request'
|
||||||
|
|
||||||
|
meta = json.dumps(meta).encode()
|
||||||
|
return b''.join(
|
||||||
|
(b'l', str(len(meta)).encode(), b':', meta, str(len(data)).encode(), b':', data, b'e')
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def decrypt_onionreq():
|
||||||
|
try:
|
||||||
|
return crypto.parse_junk(request.data)
|
||||||
|
except Exception as e:
|
||||||
|
app.logger.warning("Failed to decrypt onion request: {}".format(e))
|
||||||
|
abort(http.BAD_REQUEST)
|
||||||
|
|
||||||
|
|
||||||
@app.post("/oxen/v3/lsrpc")
|
@app.post("/oxen/v3/lsrpc")
|
||||||
@app.post("/loki/v3/lsrpc")
|
@app.post("/loki/v3/lsrpc")
|
||||||
def handle_onion_request():
|
def handle_onion_request():
|
||||||
"""
|
"""
|
||||||
Parse an onion request, handle it as a subrequest, then encrypt the subrequest result and send
|
Parse an onion request, handle it as a subrequest, then throw away the subrequest headers,
|
||||||
it back to the requestor.
|
replace the subrequest body with a json string, encrypt the final result and then pointlessly
|
||||||
|
base64 encodes the body before sending it back to the requestor.
|
||||||
|
|
||||||
|
Deprecated in favour of /v4/.
|
||||||
|
|
||||||
|
This injects a subrequest to process it then returns the result of that subrequest (as bytes).
|
||||||
|
|
||||||
|
The body must be JSON containing two always-required keys:
|
||||||
|
|
||||||
|
- "endpoint" -- the HTTP endpoint to invoke (e.g. "/room/some-room").
|
||||||
|
- "method" -- the HTTP method (e.g. "POST", "GET")
|
||||||
|
|
||||||
|
Plus, when method is POST or PUT, the required field:
|
||||||
|
|
||||||
|
- "body" -- the request body for POST/PUT requests
|
||||||
|
|
||||||
|
Optional keys that may be included are:
|
||||||
|
- "headers" -- optional dict of HTTP headers for the request. Header names are
|
||||||
|
case-insensitive (i.e. `X-Foo` and `x-FoO` are equivalent).
|
||||||
|
|
||||||
|
When returning, we invoke the subrequest and then, if it returns a 200 response code, we take
|
||||||
|
the response body, encrypt it, and then base64 the encrypted body and send that back as the
|
||||||
|
response body of the onion request.
|
||||||
|
|
||||||
|
If the subrequest returned a non-200 response code then instead of the returned body we return
|
||||||
|
`{"status_code":xxx}` (where xxx is the numeric status code) and encrypt/base64 encode that.
|
||||||
|
|
||||||
|
Response headers are completely ignored, as are bodies of non-200 responses.
|
||||||
|
|
||||||
|
This is deprecated because it amplifies request and response sizes, it doesn't allow non-json
|
||||||
|
requests, and it drops pertinent request information (such as response headers and error
|
||||||
|
bodies). Prefer v4 requests which do not have these drawbacks.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
try:
|
junk = decrypt_onionreq()
|
||||||
junk = onionparser.parse_junk(request.data)
|
return utils.encode_base64(junk.transformReply(handle_v3_onionreq_plaintext(junk.payload)))
|
||||||
except RuntimeError as e:
|
|
||||||
app.logger.warn("Failed to decrypt onion request: {}".format(e))
|
|
||||||
return Response(status=http.INTERNAL_SERVER_ERROR)
|
|
||||||
|
|
||||||
response = handle_onionreq_plaintext(junk.payload)
|
|
||||||
return base64.b64encode(junk.transformReply(response)).decode()
|
@app.post("/oxen/v4/lsrpc")
|
||||||
|
def handle_v4_onion_request():
|
||||||
|
"""
|
||||||
|
Handles a decrypted v4 onion request; this injects a subrequest to process it then returns the
|
||||||
|
result of that subrequest. In contrast to v3, it is more efficient (particularly for binary
|
||||||
|
input or output) and allows using endpoints that return headers or bodies with non-2xx response
|
||||||
|
codes.
|
||||||
|
|
||||||
|
The body of a v4 request (post-decryption) is a bencoded list containing exactly 1 or 2 byte
|
||||||
|
strings: the first byte string contains a json object containing the request metadata which has
|
||||||
|
three required fields:
|
||||||
|
|
||||||
|
- "endpoint" -- the HTTP endpoint to invoke (e.g. "/room/some-room").
|
||||||
|
- "method" -- the HTTP method (e.g. "POST", "GET")
|
||||||
|
- "headers" -- dict of HTTP headers for the request. Header names are case-insensitive (i.e.
|
||||||
|
`X-Foo` and `x-FoO` are equivalent).
|
||||||
|
|
||||||
|
Unlike v3 requests, endpoints must always start with a /. (If a legacy endpoint "whatever"
|
||||||
|
needs to be accessed through a v4 request for some reason then it can be accessed via the
|
||||||
|
"/legacy/whatever" endpoint).
|
||||||
|
|
||||||
|
The "headers" field typically carries X-SOGS-* authentication headers as well as fields like
|
||||||
|
Content-Type. Note that, unlike v3 requests, the Content-Type does *not* have any default and
|
||||||
|
should also be specified, often as `application/json`. Unlike HTTP requests, Content-Length is
|
||||||
|
not required and will be ignored if specified; the content-length is always determined from the
|
||||||
|
provided body.
|
||||||
|
|
||||||
|
The second byte string in the request, if present, is the request body in raw bytes and is
|
||||||
|
required for POST and PUT requests and must not be provided for GET/DELETE requests.
|
||||||
|
|
||||||
|
Bencoding details:
|
||||||
|
A full bencode library can be used, but the format used here is deliberately meant to be as
|
||||||
|
simple as possible to implement without a full bencode library on hand. The format of a
|
||||||
|
byte string is `N:` where N is a decimal number (e.g. `123:` starts a 123-byte string),
|
||||||
|
followed by the N bytes. A list of strings starts with `l`, contains any number of encoded
|
||||||
|
byte strings, followed by `e`. (Full bencode allows dicts, integers, and list/dict
|
||||||
|
recursion, but we do not use any of that for v4 bencoded onion requests).
|
||||||
|
|
||||||
|
For example, the request:
|
||||||
|
|
||||||
|
GET /room/some-room
|
||||||
|
Some-Header: 12345
|
||||||
|
|
||||||
|
would be encoded as:
|
||||||
|
|
||||||
|
l79:{"method":"GET","endpoint":"/room/some-room","headers":{"Some-Header":"12345"}}e
|
||||||
|
|
||||||
|
that is: a list containing a single 79-byte string. A POST request such as:
|
||||||
|
|
||||||
|
POST /some/thing
|
||||||
|
Some-Header: a
|
||||||
|
|
||||||
|
post body here
|
||||||
|
|
||||||
|
would be encoded as the two-string bencoded list:
|
||||||
|
|
||||||
|
l72:{"method":"POST","endpoint":"/some/thing","headers":{"Some-Header":"a"}}14:post body heree
|
||||||
|
^^^^^^^^72-byte request info json^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^body^^^^^
|
||||||
|
|
||||||
|
The return value of the request is always a 2-part bencoded list where the first part contains
|
||||||
|
response metadata and the second contains the response body. The response metadata is a json
|
||||||
|
object containing:
|
||||||
|
- "code" -- the numeric HTTP response code (e.g. 200, 403); and
|
||||||
|
- "headers" -- a json object of header names to values. Note that, since HTTP headers are
|
||||||
|
case-insensitive, the header names are always returned as lower-case, and we strip out the
|
||||||
|
'content-length' header (since it is already encoded in the length of the body part).
|
||||||
|
|
||||||
|
For example, a simple json request response might be the two parts:
|
||||||
|
|
||||||
|
- `{"code":200,"headers":{"content-type":"application/json"}}`
|
||||||
|
- `{"id": 123}`
|
||||||
|
|
||||||
|
encoded as:
|
||||||
|
|
||||||
|
l58:{"code":200,"headers":{"content-type":"application/json"}}11:{"id": 123}e
|
||||||
|
|
||||||
|
A more complicated request, for example for a file download, might return binary content such as:
|
||||||
|
|
||||||
|
- `{"code":200,"headers":{"content-type":"application/octet-stream","content-disposition":"attachment; filename*=UTF-8''filename.txt"}}`
|
||||||
|
- `My file contents`
|
||||||
|
|
||||||
|
i.e. encoded as `l132:{...the json above...}16:My file contentse`
|
||||||
|
|
||||||
|
Error responses (e.g. a 403) are not treated specially; that is: they still have a "code" set to
|
||||||
|
the response code and "headers" and a body part of whatever the request returned for a body).
|
||||||
|
|
||||||
|
The final value returned from the endpoint is the encrypted bencoded bytes, and these encrypted
|
||||||
|
bytes are returned directly to the client (i.e. no base64 encoding applied, unlike v3 requests).
|
||||||
|
""" # noqa: E501
|
||||||
|
|
||||||
|
# Some less-than-ideal decisions in the onion request protocol design means that we are stuck
|
||||||
|
# dealing with parsing the request body here in the internal format that is meant for storage
|
||||||
|
# server, but the *last* hop's decrypted, encoded data has to get shared by us (and is passed on
|
||||||
|
# to us in its raw, encoded form). It looks like this:
|
||||||
|
#
|
||||||
|
# [N][blob][json]
|
||||||
|
#
|
||||||
|
# where N is the size of blob (4 bytes, little endian), and json contains *both* the elements
|
||||||
|
# that were meant for the last hop (like our host/port/protocol) *and* the elements that *we*
|
||||||
|
# need to decrypt blob (specifically: "ephemeral_key" and, optionally, "enc_type" [which can be
|
||||||
|
# used to use xchacha20-poly1305 encryption instead of AES-GCM]).
|
||||||
|
#
|
||||||
|
# The parse_junk here takes care of decoding and decrypting this according to the fields *meant
|
||||||
|
# for us* in the json (which include things like the encryption type and ephemeral key):
|
||||||
|
try:
|
||||||
|
junk = crypto.parse_junk(request.data)
|
||||||
|
except RuntimeError as e:
|
||||||
|
app.logger.warning("Failed to decrypt onion request: {}".format(e))
|
||||||
|
abort(http.BAD_REQUEST)
|
||||||
|
|
||||||
|
# On the way back out we re-encrypt via the junk parser (which uses the ephemeral key and
|
||||||
|
# enc_type that were specified in the outer request). We then return that encrypted binary
|
||||||
|
# payload as-is back to the client which bounces its way through the SN path back to the client.
|
||||||
|
response = handle_v4_onionreq_plaintext(junk.payload)
|
||||||
|
return junk.transformReply(response)
|
||||||
|
|
|
@ -6,10 +6,11 @@ except ModuleNotFoundError:
|
||||||
"""Simple non-uwsgi stub that just calls the postfork function"""
|
"""Simple non-uwsgi stub that just calls the postfork function"""
|
||||||
|
|
||||||
def __init__(self, f):
|
def __init__(self, f):
|
||||||
f()
|
self.f = f
|
||||||
|
self.f()
|
||||||
|
|
||||||
def __call__(self, f):
|
def __call__(self):
|
||||||
pass
|
self.f()
|
||||||
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -1,12 +1,12 @@
|
||||||
from . import config
|
from . import config
|
||||||
from .web import app
|
from .web import app
|
||||||
from .db import psql
|
from . import db
|
||||||
from . import http
|
from . import http, utils
|
||||||
|
|
||||||
import flask
|
import flask
|
||||||
from flask import request
|
from flask import request
|
||||||
import secrets
|
import secrets
|
||||||
import base64
|
from base64 import urlsafe_b64encode
|
||||||
from hashlib import blake2b
|
from hashlib import blake2b
|
||||||
import json
|
import json
|
||||||
import psycopg
|
import psycopg
|
||||||
|
@ -38,7 +38,7 @@ def generate_file_id(data):
|
||||||
base64 chars. (Ideally would be 32, but that would result in base64 padding, so increased to 33
|
base64 chars. (Ideally would be 32, but that would result in base64 padding, so increased to 33
|
||||||
to fit perfectly).
|
to fit perfectly).
|
||||||
"""
|
"""
|
||||||
return base64.urlsafe_b64encode(
|
return urlsafe_b64encode(
|
||||||
blake2b(data, digest_size=33, salt=b"SessionFileSvr\0\0").digest()
|
blake2b(data, digest_size=33, salt=b"SessionFileSvr\0\0").digest()
|
||||||
).decode()
|
).decode()
|
||||||
|
|
||||||
|
@ -66,7 +66,7 @@ def submit_file(*, body=None, deprecated=False):
|
||||||
if not deprecated:
|
if not deprecated:
|
||||||
id = str(id) # New ids are always strings; legacy requests require an integer
|
id = str(id) # New ids are always strings; legacy requests require an integer
|
||||||
try:
|
try:
|
||||||
with psql.cursor() as cur:
|
with db.psql.cursor() as cur:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"INSERT INTO files (id, data, expiry) VALUES (%s, %s, NOW() + %s)",
|
"INSERT INTO files (id, data, expiry) VALUES (%s, %s, NOW() + %s)",
|
||||||
(id, body, config.FILE_EXPIRY),
|
(id, body, config.FILE_EXPIRY),
|
||||||
|
@ -83,11 +83,11 @@ def submit_file(*, body=None, deprecated=False):
|
||||||
return error_resp(http.INSUFFICIENT_STORAGE)
|
return error_resp(http.INSUFFICIENT_STORAGE)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
with psql.transaction(), psql.cursor() as cur:
|
with db.psql.transaction(), db.psql.cursor() as cur:
|
||||||
id = generate_file_id(body)
|
id = generate_file_id(body)
|
||||||
try:
|
try:
|
||||||
# Don't pass the data yet because we might be de-duplicating
|
# Don't pass the data yet because we might be de-duplicating
|
||||||
with psql.transaction():
|
with db.psql.transaction():
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"INSERT INTO files (id, data, expiry) VALUES (%s, '', NOW() + %s)",
|
"INSERT INTO files (id, data, expiry) VALUES (%s, '', NOW() + %s)",
|
||||||
(id, config.FILE_EXPIRY),
|
(id, config.FILE_EXPIRY),
|
||||||
|
@ -125,18 +125,14 @@ def submit_file_old():
|
||||||
)
|
)
|
||||||
return error_resp(http.PAYLOAD_TOO_LARGE)
|
return error_resp(http.PAYLOAD_TOO_LARGE)
|
||||||
|
|
||||||
# base64.b64decode is picky about padding (but not, by default, about random non-alphabet
|
body = utils.decode_base64(body)
|
||||||
# characters in the middle of the data, wtf!)
|
|
||||||
while len(body) % 4 != 0:
|
|
||||||
body += "="
|
|
||||||
body = base64.b64decode(body, validate=True)
|
|
||||||
|
|
||||||
return submit_file(body=body, deprecated=True)
|
return submit_file(body=body, deprecated=True)
|
||||||
|
|
||||||
|
|
||||||
@app.route("/file/<id>")
|
@app.get("/file/<id>")
|
||||||
def get_file(id):
|
def get_file(id):
|
||||||
with psql.cursor() as cur:
|
with db.psql.cursor() as cur:
|
||||||
cur.execute("SELECT data FROM files WHERE id = %s", (id,), binary=True)
|
cur.execute("SELECT data FROM files WHERE id = %s", (id,), binary=True)
|
||||||
row = cur.fetchone()
|
row = cur.fetchone()
|
||||||
if row:
|
if row:
|
||||||
|
@ -148,21 +144,21 @@ def get_file(id):
|
||||||
return error_resp(http.NOT_FOUND)
|
return error_resp(http.NOT_FOUND)
|
||||||
|
|
||||||
|
|
||||||
@app.route("/files/<id>")
|
@app.get("/files/<id>")
|
||||||
def get_file_old(id):
|
def get_file_old(id):
|
||||||
with psql.cursor() as cur:
|
with db.psql.cursor() as cur:
|
||||||
cur.execute("SELECT data FROM files WHERE id = %s", (id,), binary=True)
|
cur.execute("SELECT data FROM files WHERE id = %s", (id,), binary=True)
|
||||||
row = cur.fetchone()
|
row = cur.fetchone()
|
||||||
if row:
|
if row:
|
||||||
return json_resp({"status_code": 200, "result": base64.b64encode(row[0]).decode()})
|
return json_resp({"status_code": 200, "result": utils.encode_base64(row[0])})
|
||||||
else:
|
else:
|
||||||
app.logger.warn("File '{}' does not exist".format(id))
|
app.logger.warn("File '{}' does not exist".format(id))
|
||||||
return error_resp(http.NOT_FOUND)
|
return error_resp(http.NOT_FOUND)
|
||||||
|
|
||||||
|
|
||||||
@app.route("/file/<id>/info")
|
@app.get("/file/<id>/info")
|
||||||
def get_file_info(id):
|
def get_file_info(id):
|
||||||
with psql.cursor() as cur:
|
with db.psql.cursor() as cur:
|
||||||
cur.execute("SELECT length(data), uploaded, expiry FROM files WHERE id = %s", (id,))
|
cur.execute("SELECT length(data), uploaded, expiry FROM files WHERE id = %s", (id,))
|
||||||
row = cur.fetchone()
|
row = cur.fetchone()
|
||||||
if row:
|
if row:
|
||||||
|
@ -174,16 +170,16 @@ def get_file_info(id):
|
||||||
return error_resp(http.NOT_FOUND)
|
return error_resp(http.NOT_FOUND)
|
||||||
|
|
||||||
|
|
||||||
@app.route("/session_version")
|
@app.get("/session_version")
|
||||||
def get_session_version():
|
def get_session_version():
|
||||||
platform = request.args["platform"]
|
platform = request.args.get("platform")
|
||||||
|
|
||||||
if platform not in ("desktop", "android", "ios"):
|
if platform not in ("desktop", "android", "ios"):
|
||||||
app.logger.warn("Invalid session platform '{}'".format(platform))
|
app.logger.warn("Invalid session platform '{}'".format(platform))
|
||||||
return error_resp(http.NOT_FOUND)
|
return error_resp(http.NOT_FOUND)
|
||||||
project = "oxen-io/session-" + platform
|
project = "oxen-io/session-" + platform
|
||||||
|
|
||||||
with psql.cursor() as cur:
|
with db.psql.cursor() as cur:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"""
|
"""
|
||||||
SELECT version, updated FROM release_versions
|
SELECT version, updated FROM release_versions
|
||||||
|
|
|
@ -0,0 +1,95 @@
|
||||||
|
from .web import app
|
||||||
|
from . import http
|
||||||
|
|
||||||
|
from flask import request
|
||||||
|
from io import BytesIO
|
||||||
|
import traceback
|
||||||
|
from typing import Optional, Union
|
||||||
|
|
||||||
|
|
||||||
|
def make_subrequest(
|
||||||
|
method: str,
|
||||||
|
path: str,
|
||||||
|
*,
|
||||||
|
headers={},
|
||||||
|
content_type: Optional[str] = None,
|
||||||
|
body: Optional[Union[bytes, memoryview]] = None,
|
||||||
|
json: Optional[Union[dict, list]] = None,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Makes a subrequest from the given parameters, returns the response object and a dict of
|
||||||
|
lower-case response headers keys to header values.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
method - the HTTP method, e.g. GET or POST
|
||||||
|
path - the request path (optionally including a query string)
|
||||||
|
headers - dict of HTTP headers for the request
|
||||||
|
content_type - the content-type of the request (for POST/PUT methods)
|
||||||
|
body - the bytes content of the body of a POST/PUT method. If specified then content_type will
|
||||||
|
default to 'application/octet-stream'.
|
||||||
|
json - a json value to dump as the body of the request. If specified then content_type will
|
||||||
|
default to 'applicaton/json'.
|
||||||
|
"""
|
||||||
|
|
||||||
|
http_headers = {'HTTP_{}'.format(h.upper().replace('-', '_')): v for h, v in headers.items()}
|
||||||
|
|
||||||
|
if content_type is None:
|
||||||
|
if 'HTTP_CONTENT_TYPE' in http_headers:
|
||||||
|
content_type = http_headers['HTTP_CONTENT_TYPE']
|
||||||
|
elif body is not None:
|
||||||
|
content_type = 'application/octet-stream'
|
||||||
|
elif json is not None:
|
||||||
|
content_type = 'application/json'
|
||||||
|
else:
|
||||||
|
content_type = ''
|
||||||
|
|
||||||
|
for x in ('HTTP_CONTENT_TYPE', 'HTTP_CONTENT_LENGTH'):
|
||||||
|
if x in http_headers:
|
||||||
|
del http_headers[x]
|
||||||
|
|
||||||
|
if body is None:
|
||||||
|
if json is not None:
|
||||||
|
from json import dumps
|
||||||
|
|
||||||
|
body = dumps(json, separators=(',', ':')).encode()
|
||||||
|
else:
|
||||||
|
body = b''
|
||||||
|
|
||||||
|
body_input = BytesIO(body)
|
||||||
|
content_length = len(body)
|
||||||
|
|
||||||
|
if '?' in path:
|
||||||
|
path, query_string = path.split('?', 1)
|
||||||
|
else:
|
||||||
|
query_string = ''
|
||||||
|
|
||||||
|
# Set up the wsgi environ variables for the subrequest (see PEP 0333)
|
||||||
|
subreq_env = {
|
||||||
|
**request.environ,
|
||||||
|
"REQUEST_METHOD": method,
|
||||||
|
"PATH_INFO": path,
|
||||||
|
"QUERY_STRING": query_string,
|
||||||
|
"CONTENT_TYPE": content_type,
|
||||||
|
"CONTENT_LENGTH": content_length,
|
||||||
|
**http_headers,
|
||||||
|
'wsgi.input': body_input,
|
||||||
|
'flask._preserve_context': False,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
app.logger.debug(f"Initiating sub-request for {method} {path}")
|
||||||
|
with app.request_context(subreq_env):
|
||||||
|
response = app.full_dispatch_request()
|
||||||
|
if response.status_code != http.OK:
|
||||||
|
app.logger.warning(
|
||||||
|
f"Sub-request for {method} {path} returned status {response.status_code}"
|
||||||
|
)
|
||||||
|
return response, {
|
||||||
|
k.lower(): v
|
||||||
|
for k, v in response.get_wsgi_headers(subreq_env)
|
||||||
|
if k.lower() != 'content-length'
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
app.logger.warning(f"Sub-request for {method} {path} failed: {traceback.format_exc()}")
|
||||||
|
raise
|
|
@ -0,0 +1,35 @@
|
||||||
|
from typing import Tuple
|
||||||
|
import base64
|
||||||
|
|
||||||
|
|
||||||
|
def bencode_consume_string(body: memoryview) -> Tuple[memoryview, memoryview]:
|
||||||
|
"""
|
||||||
|
Parses a bencoded byte string from the beginning of `body`. Returns a pair of memoryviews on
|
||||||
|
success: the first is the string byte data; the second is the remaining data (i.e. after the
|
||||||
|
consumed string).
|
||||||
|
Raises ValueError on parse failure.
|
||||||
|
"""
|
||||||
|
pos = 0
|
||||||
|
print(f"body: {body.tobytes()}")
|
||||||
|
while pos < len(body) and 0x30 <= body[pos] <= 0x39: # 1+ digits
|
||||||
|
pos += 1
|
||||||
|
if pos == 0 or pos >= len(body) or body[pos] != 0x3A: # 0x3a == ':'
|
||||||
|
raise ValueError("Invalid string bencoding: did not find `N:` length prefix")
|
||||||
|
|
||||||
|
strlen = int(body[0:pos]) # parse the digits as a base-10 integer
|
||||||
|
pos += 1 # skip the colon
|
||||||
|
if pos + strlen > len(body):
|
||||||
|
raise ValueError("Invalid string bencoding: length exceeds buffer")
|
||||||
|
return body[pos : pos + strlen], body[pos + strlen :]
|
||||||
|
|
||||||
|
|
||||||
|
def encode_base64(data: bytes):
|
||||||
|
return base64.b64encode(data).decode()
|
||||||
|
|
||||||
|
|
||||||
|
def decode_base64(b64: str):
|
||||||
|
"""Decodes a base64 value with or without padding."""
|
||||||
|
# Accept unpadded base64 by appending padding; b64decode won't accept it otherwise
|
||||||
|
if 2 <= len(b64) % 4 <= 3 and not b64.endswith('='):
|
||||||
|
b64 += '=' * (4 - len(b64) % 4)
|
||||||
|
return base64.b64decode(b64, validate=True)
|
|
@ -1 +0,0 @@
|
||||||
Subproject commit bc373fcb0915e0378a898762e239266409d62f0c
|
|
|
@ -0,0 +1,71 @@
|
||||||
|
import pytest
|
||||||
|
import os
|
||||||
|
|
||||||
|
from fileserver import config
|
||||||
|
|
||||||
|
config.pgsql_connect_opts = {"defer": True}
|
||||||
|
|
||||||
|
from fileserver import web # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def pytest_addoption(parser):
|
||||||
|
parser.addoption(
|
||||||
|
"--pgsql",
|
||||||
|
type=str,
|
||||||
|
help='Use the given postgresql database connect string for testing. '
|
||||||
|
'E.g. "dbname=test user=joe" or "postgresql://..."',
|
||||||
|
required=True,
|
||||||
|
)
|
||||||
|
parser.addoption(
|
||||||
|
"--no-drop-schema",
|
||||||
|
action="store_true",
|
||||||
|
default=False,
|
||||||
|
help="Don't clean up the final test schema; typically used with --maxfail=1",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def db_conn(request):
|
||||||
|
from fileserver import db as db_
|
||||||
|
|
||||||
|
pgsql = request.config.getoption("--pgsql")
|
||||||
|
web.app.logger.warning(f"using postgresql {pgsql}")
|
||||||
|
|
||||||
|
config.pgsql_connect_opts = {"conninfo": pgsql}
|
||||||
|
db_.pg_connect()
|
||||||
|
db_.psql = db_.psql_pool.getconn()
|
||||||
|
|
||||||
|
yield db_.psql
|
||||||
|
|
||||||
|
web.app.logger.warning("closing db")
|
||||||
|
if not request.config.getoption("--no-drop-schema"):
|
||||||
|
web.app.logger.warning("DROPPING SCHEMA")
|
||||||
|
with db_.psql.cursor() as cur:
|
||||||
|
cur.execute("DROP SCHEMA sfs_tests CASCADE")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def db(request, db_conn):
|
||||||
|
"""
|
||||||
|
Import this fixture to get a wiped, re-initialized database for db.engine. The actual fixture
|
||||||
|
value is the db module itself (so typically you don't import it at all but instead get it
|
||||||
|
through this fixture, which also creates an empty db for you).
|
||||||
|
"""
|
||||||
|
|
||||||
|
with db_conn.transaction(), db_conn.cursor() as cur, open(
|
||||||
|
os.path.dirname(__file__) + "/../schema.pgsql", "r"
|
||||||
|
) as schema:
|
||||||
|
cur.execute("DROP SCHEMA IF EXISTS sfs_tests CASCADE")
|
||||||
|
cur.execute("CREATE SCHEMA IF NOT EXISTS sfs_tests")
|
||||||
|
cur.execute("SET search_path TO sfs_tests")
|
||||||
|
cur.execute(schema.read())
|
||||||
|
|
||||||
|
return db_conn
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def client():
|
||||||
|
"""Yields an flask test client for the app that can be used to make test requests"""
|
||||||
|
|
||||||
|
with web.app.test_client() as client:
|
||||||
|
yield client
|
|
@ -0,0 +1,241 @@
|
||||||
|
from fileserver.web import app
|
||||||
|
from fileserver import crypto, db, utils
|
||||||
|
from nacl.bindings import (
|
||||||
|
crypto_scalarmult,
|
||||||
|
crypto_aead_xchacha20poly1305_ietf_encrypt,
|
||||||
|
crypto_aead_xchacha20poly1305_ietf_decrypt,
|
||||||
|
)
|
||||||
|
from Cryptodome.Cipher import AES
|
||||||
|
import nacl.utils
|
||||||
|
import nacl.hashlib
|
||||||
|
import struct
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
|
||||||
|
from nacl.public import PrivateKey
|
||||||
|
|
||||||
|
# ephemeral X25519 keypair for use in tests
|
||||||
|
a = PrivateKey(bytes.fromhex('ec32fb5766cf52b1b5d7b0bff08e29f5c0c58ca19beaf6a5c7d3dd8ac7ced963'))
|
||||||
|
A = a.public_key
|
||||||
|
assert A.encode().hex() == 'd79a50b82ba8ca665f854382b42ba159efd16eef87409e97a8d07395b9492928'
|
||||||
|
|
||||||
|
# For xchacha20 we use the libsodium recommended shared key of H(aB || A || B), where H(.) is
|
||||||
|
# 32-byte Blake2B
|
||||||
|
shared_xchacha20_key = nacl.hashlib.blake2b(
|
||||||
|
crypto_scalarmult(a.encode(), crypto.privkey.public_key.encode())
|
||||||
|
+ A.encode()
|
||||||
|
+ crypto.privkey.public_key.encode(),
|
||||||
|
digest_size=32,
|
||||||
|
).digest()
|
||||||
|
|
||||||
|
# AES-GCM onion requests were implemented using the somewhat weaker shared key of just aB:
|
||||||
|
shared_aes_key = crypto_scalarmult(a.encode(), crypto.privkey.public_key.encode())
|
||||||
|
|
||||||
|
|
||||||
|
def build_payload(inner_json, inner_body=None, *, v, enc_type, outer_json_extra={}):
|
||||||
|
"""Encrypt and encode a payload for fileserver"""
|
||||||
|
|
||||||
|
if not isinstance(inner_json, bytes):
|
||||||
|
inner_json = json.dumps(inner_json).encode()
|
||||||
|
|
||||||
|
if v == 3:
|
||||||
|
assert inner_body is None
|
||||||
|
inner_data = inner_json
|
||||||
|
elif v == 4:
|
||||||
|
inner_data = b''.join(
|
||||||
|
(
|
||||||
|
b'l',
|
||||||
|
str(len(inner_json)).encode(),
|
||||||
|
b':',
|
||||||
|
inner_json,
|
||||||
|
*(() if inner_body is None else (str(len(inner_body)).encode(), b':', inner_body)),
|
||||||
|
b'e',
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise RuntimeError(f"invalid payload v{v}")
|
||||||
|
|
||||||
|
inner_enc = ()
|
||||||
|
if enc_type in ("xchacha20", "xchacha20-poly1305"):
|
||||||
|
# For xchacha20 we stick the nonce to the beginning of the encrypted blob
|
||||||
|
nonce = nacl.utils.random(24)
|
||||||
|
inner_enc = (
|
||||||
|
nonce,
|
||||||
|
crypto_aead_xchacha20poly1305_ietf_encrypt(
|
||||||
|
inner_data, aad=None, nonce=nonce, key=shared_xchacha20_key
|
||||||
|
),
|
||||||
|
)
|
||||||
|
elif enc_type in ("aes-gcm", "gcm"):
|
||||||
|
# For aes-gcm we stick the iv on the beginning of the encrypted blob and the mac tag on the
|
||||||
|
# end of it
|
||||||
|
iv = nacl.utils.random(12)
|
||||||
|
cipher = AES.new(shared_aes_key, AES.MODE_GCM, iv)
|
||||||
|
enc, mac = cipher.encrypt_and_digest(inner_data)
|
||||||
|
inner_enc = (iv, enc, mac)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Invalid enc_type: {enc_type}")
|
||||||
|
|
||||||
|
# The outer request is in storage server onion request format:
|
||||||
|
# [N][junk]{json}
|
||||||
|
# where we load the fields for the last hop *and* the fields for fileserver into the json.
|
||||||
|
outer_json = {
|
||||||
|
"host": "localhost",
|
||||||
|
"port": 80,
|
||||||
|
"protocol": "http",
|
||||||
|
"target": f"/oxen/v{v}/lsrpc",
|
||||||
|
"ephemeral_key": A.encode().hex(),
|
||||||
|
"enc_type": enc_type,
|
||||||
|
**outer_json_extra,
|
||||||
|
}
|
||||||
|
return b''.join(
|
||||||
|
(
|
||||||
|
struct.pack('<i', sum(len(x) for x in inner_enc)),
|
||||||
|
*inner_enc,
|
||||||
|
json.dumps(outer_json).encode(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def decrypt_reply(data, *, v, enc_type):
|
||||||
|
"""
|
||||||
|
Parses a reply; returns the json metadata and the body. Note for v3 that there is only json;
|
||||||
|
body will always be None.
|
||||||
|
"""
|
||||||
|
if v == 3:
|
||||||
|
data = utils.decode_base64(data)
|
||||||
|
|
||||||
|
if enc_type in ("xchacha20", "xchacha20-poly1305"):
|
||||||
|
assert len(data) > 24
|
||||||
|
nonce, enc = data[:24], data[24:]
|
||||||
|
data = crypto_aead_xchacha20poly1305_ietf_decrypt(
|
||||||
|
enc, aad=None, nonce=nonce, key=shared_xchacha20_key
|
||||||
|
)
|
||||||
|
elif enc_type in ("aes-gcm", "gcm"):
|
||||||
|
assert len(data) > 28
|
||||||
|
iv, enc, mac = data[:12], data[12:-16], data[-16:]
|
||||||
|
cipher = AES.new(shared_aes_key, AES.MODE_GCM, iv)
|
||||||
|
data = cipher.decrypt_and_verify(enc, mac)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Invalid enc_type: {enc_type}")
|
||||||
|
|
||||||
|
body = None
|
||||||
|
|
||||||
|
if v == 4:
|
||||||
|
assert (data[:1], data[-1:]) == (b'l', b'e')
|
||||||
|
data = memoryview(data)[1:-1]
|
||||||
|
json_data, data = utils.bencode_consume_string(data)
|
||||||
|
json_ = json.loads(json_data.tobytes())
|
||||||
|
if data:
|
||||||
|
body, data = utils.bencode_consume_string(data)
|
||||||
|
assert len(data) == 0
|
||||||
|
body = body.tobytes()
|
||||||
|
elif v == 3:
|
||||||
|
json_ = json.loads(data)
|
||||||
|
|
||||||
|
return json_, body
|
||||||
|
|
||||||
|
|
||||||
|
def update_session_desktop_version():
|
||||||
|
"""
|
||||||
|
Fileserver errors if the db update version is more than 24 hours old, so update it to a fake
|
||||||
|
version for testing.
|
||||||
|
"""
|
||||||
|
|
||||||
|
with db.psql.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
"UPDATE release_versions SET version = %s, updated = NOW() WHERE project = %s",
|
||||||
|
('v1.2.3', 'oxen-io/session-desktop'),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_v3(client):
|
||||||
|
update_session_desktop_version()
|
||||||
|
|
||||||
|
# Construct an onion request for /room/test-room
|
||||||
|
req = {'method': 'GET', 'endpoint': 'session_version?platform=desktop'}
|
||||||
|
data = build_payload(req, v=3, enc_type="xchacha20")
|
||||||
|
|
||||||
|
r = client.post("/loki/v3/lsrpc", data=data)
|
||||||
|
|
||||||
|
assert r.status_code == 200
|
||||||
|
|
||||||
|
v = decrypt_reply(r.data, v=3, enc_type="xchacha20")[0]
|
||||||
|
|
||||||
|
assert -1 < time.time() - v.pop('updated') < 1
|
||||||
|
assert v == {'status_code': 200, 'result': 'v1.2.3'}
|
||||||
|
|
||||||
|
|
||||||
|
def test_v4(client):
|
||||||
|
update_session_desktop_version()
|
||||||
|
|
||||||
|
req = {'method': 'GET', 'endpoint': '/session_version?platform=desktop'}
|
||||||
|
data = build_payload(req, v=4, enc_type="xchacha20")
|
||||||
|
|
||||||
|
r = client.post("/oxen/v4/lsrpc", data=data)
|
||||||
|
|
||||||
|
assert r.status_code == 200
|
||||||
|
|
||||||
|
info, body = decrypt_reply(r.data, v=4, enc_type="xchacha20")
|
||||||
|
|
||||||
|
assert info == {'code': 200, 'headers': {'content-type': 'application/json'}}
|
||||||
|
|
||||||
|
v = json.loads(body)
|
||||||
|
assert -1 < time.time() - v.pop('updated') < 1
|
||||||
|
assert v == {'status_code': 200, 'result': 'v1.2.3'}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/test_v4_post_body")
|
||||||
|
def v4_post_body():
|
||||||
|
from flask import request, jsonify, Response
|
||||||
|
|
||||||
|
if request.json is not None:
|
||||||
|
return jsonify({"json": request.json})
|
||||||
|
print(f"rd: {request.data}")
|
||||||
|
return Response(
|
||||||
|
f"not json ({request.content_type}): {request.data.decode()}".encode(),
|
||||||
|
mimetype='text/plain',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_v4_post_body(client):
|
||||||
|
req = {'method': 'POST', 'endpoint': '/test_v4_post_body'}
|
||||||
|
content = b'test data'
|
||||||
|
req['headers'] = {'content-type': 'text/plain'}
|
||||||
|
|
||||||
|
data = build_payload(req, content, v=4, enc_type="xchacha20")
|
||||||
|
|
||||||
|
r = client.post("/oxen/v4/lsrpc", data=data)
|
||||||
|
|
||||||
|
assert r.status_code == 200
|
||||||
|
|
||||||
|
info, body = decrypt_reply(r.data, v=4, enc_type="xchacha20")
|
||||||
|
|
||||||
|
assert info == {'code': 200, 'headers': {'content-type': 'text/plain; charset=utf-8'}}
|
||||||
|
assert body == b'not json (text/plain): test data'
|
||||||
|
|
||||||
|
# Now try with json:
|
||||||
|
test_json = {"test": ["json", None], "1": 23}
|
||||||
|
content = json.dumps(test_json).encode()
|
||||||
|
req['headers'] = {'content-type': 'application/json'}
|
||||||
|
|
||||||
|
data = build_payload(req, content, v=4, enc_type="xchacha20")
|
||||||
|
r = client.post("/oxen/v4/lsrpc", data=data)
|
||||||
|
|
||||||
|
assert r.status_code == 200
|
||||||
|
|
||||||
|
info, body = decrypt_reply(r.data, v=4, enc_type="xchacha20")
|
||||||
|
|
||||||
|
assert info == {'code': 200, 'headers': {'content-type': 'application/json'}}
|
||||||
|
assert json.loads(body) == {"json": test_json}
|
||||||
|
|
||||||
|
# Now try with json, but with content-type set to something else (this should avoid the json
|
||||||
|
req['headers'] = {'content-type': 'x-omg/all-your-base'}
|
||||||
|
data = build_payload(req, content, v=4, enc_type="xchacha20")
|
||||||
|
r = client.post("/oxen/v4/lsrpc", data=data)
|
||||||
|
|
||||||
|
assert r.status_code == 200
|
||||||
|
|
||||||
|
info, body = decrypt_reply(r.data, v=4, enc_type="xchacha20")
|
||||||
|
|
||||||
|
assert info == {'code': 200, 'headers': {'content-type': 'text/plain; charset=utf-8'}}
|
||||||
|
assert body == b'not json (x-omg/all-your-base): ' + content
|
Loading…
Reference in New Issue