Add v4 onion requests

Copied over from sogs (and slightly modified to remove optional
sogs-specific bits).

Adds a pytest suite with onion request tests.

Also migrates to the deb.oxen.io oxenmq and pyonionreq python packages.

.drone.jsonnet

@@ -1,6 +1,74 @@
 local docker_base = 'registry.oxen.rocks/lokinet-ci-';
 local apt_get_quiet = 'apt-get -o=Dpkg::Use-Pty=0 -q';
 
+local default_deps = [
+  'python3',
+  'python3-pytest',
+  'python3-coloredlogs',
+  'python3-flask',
+  'python3-pycryptodome',
+  'python3-nacl',
+  'python3-requests',
+  'python3-pyonionreq',
+  'python3-oxenmq',
+];
+
+// Regular build on a debian-like system:
+local debian_pipeline(name,
+                      image,
+                      arch='amd64',
+                      deps=default_deps,
+                      before_pytest=[],
+                      pytest_opts='',
+                      extra_cmds=[],
+                      services=[],
+                      allow_fail=false) = {
+  kind: 'pipeline',
+  type: 'docker',
+  name: name,
+  platform: { arch: arch },
+  trigger: { branch: { exclude: ['debian/*', 'ubuntu/*'] } },
+  steps: [
+    {
+      name: '🐍 pytest',
+      image: image,
+      pull: 'always',
+      [if allow_fail then 'failure']: 'ignore',
+      commands: [
+                  'echo "Running on ${DRONE_STAGE_MACHINE}"',
+                  'echo "man-db man-db/auto-update boolean false" | debconf-set-selections',
+                  apt_get_quiet + ' update',
+                  apt_get_quiet + ' install -y eatmydata',
+                  'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y lsb-release',
+                  'cp contrib/deb.oxen.io.gpg /etc/apt/trusted.gpg.d',
+                  'echo deb http://deb.oxen.io $$(lsb_release -sc) main >/etc/apt/sources.list.d/oxen.list',
+                  'eatmydata ' + apt_get_quiet + ' update',
+                  'eatmydata ' + apt_get_quiet + ' dist-upgrade -y',
+                  'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y ' + std.join(' ', deps),
+                  'cp fileserver/config.py.sample fileserver/config.py',
+                ] + before_pytest + [
+                  'PYTHONPATH=. python3 -mpytest -vv --color=yes ' + pytest_opts,
+                ]
+                + extra_cmds,
+    },
+  ],
+  services: services,
+};
+
+local debian_pg_pipeline(name, image, pg_tag='bullseye') = debian_pipeline(
+  name,
+  image,
+  deps=default_deps + ['python3-pip', 'postgresql-client'],
+  services=[
+    { name: 'pg', image: 'postgres:bullseye', environment: { POSTGRES_USER: 'ci', POSTGRES_PASSWORD: 'ci' } },
+  ],
+  before_pytest=[
+    'pip3 install psycopg psycopg-pool',
+    'for i in $(seq 0 30); do if pg_isready -d ci -h pg -U ci -t 1; then break; fi; if [ "$i" = 30 ]; then echo "Timeout waiting for postgresql" >&2; exit 1; fi; sleep 1; done',
+  ],
+  pytest_opts='--pgsql "postgresql://ci:ci@pg/ci"'
+);
+
 [
   {
     name: 'Lint checks',
@@ -28,4 +96,6 @@ local apt_get_quiet = 'apt-get -o=Dpkg::Use-Pty=0 -q';
       },
     ],
   },
+
+  debian_pg_pipeline('PostgreSQL 13/bullseye', docker_base + 'debian-stable', pg_tag='13-bullseye'),
 ]

.flake8

@@ -1,3 +1,4 @@
 [flake8]
 max-line-length = 100
 exclude=libonionrequests,fileserver/config.py
+extend-ignore = E203  # See https://github.com/psf/black/issues/315

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "libonionrequests"]
-	path = libonionrequests
-	url = https://github.com/majestrate/libonionrequests.git

README.md

@@ -21,10 +21,11 @@ Debian/Ubuntu).
 - psycopg_pool
 - requests
 - uwsgidecorators
+- oxenmq
+- pyonionreq
 
-Additionally you need to build the Oxen Project's pyoxenmq and pylibonionreq.  This repository links
-to them as submodules; `make` will build them locally for simple setups (proper deb packaging of
-those libs is still a TODO).
+(The last two are Oxen projects that either need to be installed manually, or via the Oxen deb
+repository).
 
 ### WSGI request handler
 

fileserver/cleanup.py

@@ -1,5 +1,5 @@
 from .web import app
-from .db import psql
+from . import db
 from .timer import timer
 from .stats import log_stats
 
@@ -11,7 +11,7 @@ last_stats_printed = None
 
 @timer(15)
 def periodic(signum):
-    with app.app_context(), psql.cursor() as cur:
+    with app.app_context(), db.psql.cursor() as cur:
         app.logger.debug("Cleaning up expired files")
         cur.execute("DELETE FROM files WHERE expiry <= NOW()")
 

fileserver/crypto.py

@@ -1,5 +1,6 @@
 import nacl.public
 import os
+import pyonionreq.junk
 
 from .web import app
 
@@ -16,6 +17,9 @@ else:
     with open("key_x25519", "wb") as f:
         f.write(privkey.encode())
 
+_junk_parser = pyonionreq.junk.Parser(privkey=privkey.encode(), pubkey=privkey.public_key.encode())
+parse_junk = _junk_parser.parse_junk
+
 app.logger.info(
     "File server pubkey: {}".format(
         privkey.public_key.encode(encoder=nacl.encoding.HexEncoder).decode()

fileserver/db.py

@@ -10,8 +10,14 @@ from werkzeug.local import LocalProxy
 @postfork
 def pg_connect():
     global psql_pool
+
+    # Test suite sets this to handle the connection itself:
+    if 'defer' in config.pgsql_connect_opts:
+        return
+
+    conninfo = config.pgsql_connect_opts.pop('conninfo', '')
     psql_pool = ConnectionPool(
-        min_size=2, max_size=32, kwargs={**config.pgsql_connect_opts, "autocommit": True}
+        conninfo, min_size=2, max_size=32, kwargs={**config.pgsql_connect_opts, "autocommit": True}
     )
     psql_pool.wait()
 

fileserver/http.py

@@ -1,3 +1,5 @@
+OK = 200
+
 # error status codes:
 BAD_REQUEST = 400
 NOT_FOUND = 404

fileserver/onion_req.py

@@ -1,131 +1,269 @@
-from flask import request, Response
-import base64
+from flask import request, abort
 import json
-from io import BytesIO
-import pyonionreq.junk
 
 from .web import app
-from . import http
-from . import crypto
-
-import traceback
-
-onionparser = pyonionreq.junk.Parser(
-    pubkey=crypto.privkey.public_key.encode(), privkey=crypto.privkey.encode()
-)
+from . import crypto, http, utils
+from .subrequest import make_subrequest
 
 
-def handle_onionreq_plaintext(body):
-    """
-    Handles a decrypted onion request; this injects a subrequest to process it then returns the
-    result of that subrequest (as bytes).
-
-    Note that this does not throw: if errors occur we map them into "success" responses with a body
-    of {"status_code":xxx} as onion requests have no ability at all to signal a request failure.
-    """
+def handle_v3_onionreq_plaintext(body):
     try:
-        if body.startswith(b"{"):
-            # JSON input
-            req = json.loads(body)
-            endpoint, method = req["endpoint"], req["method"]
-            subreq_headers = {k.lower(): v for k, v in req.get("headers", {}.items()).items()}
+        if not body.startswith(b'{'):
+            raise RuntimeError("Invalid v3 onion request body: expected JSON object")
 
-            if method in http.BODY_METHODS:
-                if "body_binary" in req:
-                    subreq_body = base64.b64decode(req["body_binary"], validate=True)
-                else:
-                    subreq_body = req.get("body", "").encode()
-                ct = subreq_headers.pop(
-                    "content-type",
-                    "application/octet-stream" if "body_binary" in req else "application/json",
-                )
-                cl = len(subreq_body)
-            else:
-                subreq_body = b""
-                # Android bug workaround: Android Session (at least up to v1.11.12) sends a body on
-                # GET requests with a 4-character string "null" when it should send no body.
-                if "body" in req and len(req["body"]) == 4 and req["body"] == "null":
-                    del req["body"]
-
-                if "body" in req and len(req["body"]) or "body_binary" in req:
-                    raise RuntimeError(
-                        "Invalid {} {} request: request must not contain a body".format(
-                            method, endpoint
-                        )
-                    )
-                ct, cl = "", ""
-
-            for h in ("content-type", "content-length"):
-                if h in subreq_headers:
-                    del subreq_headers[h]
-
-        elif body.startswith(b"d"):
-            raise RuntimeError("Bencoded onion requests not implemented yet")
+        req = json.loads(body)
+        endpoint, method = req['endpoint'], req['method']
+        subreq_headers = {k.lower(): v for k, v in req.get('headers', {}).items()}
 
+        if method in http.BODY_METHODS:
+            subreq_body = req.get('body', '').encode()
         else:
-            raise RuntimeError(
-                "Invalid onion request body: expected JSON object or a bt-encoded dict"
-            )
+            subreq_body = b''
+            # Android bug workaround: Android Session (at least up to v1.11.12) sends a body on
+            # GET requests with a 4-character string "null" when it should send no body.
+            if 'body' in req and len(req['body']) == 4 and req['body'] == 'null':
+                del req['body']
 
-        if "?" in endpoint:
-            endpoint, query_string = endpoint.split("?", 1)
-        else:
-            query_string = ""
-
-        # Set up the wsgi environ variables for the subrequest (see PEP 0333)
-        subreq_env = {
-            **request.environ,
-            "REQUEST_METHOD": method,
-            "PATH_INFO": endpoint,
-            "QUERY_STRING": query_string,
-            "CONTENT_TYPE": ct,
-            "CONTENT_LENGTH": cl,
-            **{"HTTP_{}".format(h.upper().replace("-", "_")): v for h, v in subreq_headers.items()},
-            "wsgi.input": BytesIO(subreq_body),
-        }
-
-        try:
-            with app.request_context(subreq_env):
-                response = app.full_dispatch_request()
-            if response.status_code == 200:
-                data = response.get_data()
-                app.logger.debug(
-                    "Onion sub-request for {} returned success, {} bytes".format(
-                        endpoint, len(data)
+            if 'body' in req and req['body']:
+                raise RuntimeError(
+                    "Invalid {} {} request: request must not contain a body".format(
+                        method, endpoint
                     )
                 )
-                return data
-            app.logger.warn(
-                "Onion sub-request for {} {} returned status {}".format(
-                    method, endpoint, response.status_code
-                )
+
+        if not endpoint.startswith('/'):
+            endpoint = '/' + endpoint
+
+        response, _headers = make_subrequest(
+            method,
+            endpoint,
+            headers=subreq_headers,
+            body=subreq_body,
+            content_type='application/json',
+        )
+
+        if response.status_code == http.OK:
+            data = response.get_data()
+            app.logger.debug(
+                f"Onion sub-request for {endpoint} returned success, {len(data)} bytes"
             )
-            return json.dumps({"status_code": response.status_code}).encode()
-        except Exception:
-            app.logger.warn(
-                "Onion sub-request for {} {} failed: {}".format(
-                    method, endpoint, traceback.format_exc()
-                )
-            )
-            return json.dumps({"status_code": http.BAD_GATEWAY}).encode()
+            return data
+        return json.dumps({'status_code': response.status_code}).encode()
+
     except Exception as e:
-        app.logger.warn("Invalid onion request: {}".format(e))
-        return json.dumps({"status_code": http.BAD_REQUEST}).encode()
+        app.logger.warning("Invalid onion request: {}".format(e))
+        return json.dumps({'status_code': http.BAD_REQUEST}).encode()
+
+
+def handle_v4_onionreq_plaintext(body):
+    try:
+        if not (body.startswith(b'l') and body.endswith(b'e')):
+            raise RuntimeError("Invalid onion request body: expected bencoded list")
+
+        belems = memoryview(body)[1:-1]
+
+        # Metadata json; this element is always required:
+        meta, belems = utils.bencode_consume_string(belems)
+
+        meta = json.loads(meta.tobytes())
+
+        # Then we can have a second optional string containing the body:
+        if len(belems) > 1:
+            subreq_body, belems = utils.bencode_consume_string(belems)
+            if len(belems):
+                raise RuntimeError("Invalid v4 onion request: found more than 2 parts")
+        else:
+            subreq_body = b''
+
+        method, endpoint = meta['method'], meta['endpoint']
+        if not endpoint.startswith('/'):
+            raise RuntimeError("Invalid v4 onion request: endpoint must start with /")
+
+        response, headers = make_subrequest(
+            method, endpoint, headers=meta.get('headers', {}), body=subreq_body
+        )
+
+        data = response.get_data()
+        app.logger.debug(
+            f"Onion sub-request for {endpoint} returned {response.status_code}, {len(data)} bytes"
+        )
+
+        meta = {'code': response.status_code, 'headers': headers}
+
+    except Exception as e:
+        app.logger.warning("Invalid v4 onion request: {}".format(e))
+        meta = {'code': http.BAD_REQUEST, 'headers': {'content-type': 'text/plain; charset=utf-8'}}
+        data = b'Invalid v4 onion request'
+
+    meta = json.dumps(meta).encode()
+    return b''.join(
+        (b'l', str(len(meta)).encode(), b':', meta, str(len(data)).encode(), b':', data, b'e')
+    )
+
+
+def decrypt_onionreq():
+    try:
+        return crypto.parse_junk(request.data)
+    except Exception as e:
+        app.logger.warning("Failed to decrypt onion request: {}".format(e))
+    abort(http.BAD_REQUEST)
 
 
 @app.post("/oxen/v3/lsrpc")
 @app.post("/loki/v3/lsrpc")
 def handle_onion_request():
     """
-    Parse an onion request, handle it as a subrequest, then encrypt the subrequest result and send
-    it back to the requestor.
+    Parse an onion request, handle it as a subrequest, then throw away the subrequest headers,
+    replace the subrequest body with a json string, encrypt the final result and then pointlessly
+    base64 encodes the body before sending it back to the requestor.
+
+    Deprecated in favour of /v4/.
+
+    This injects a subrequest to process it then returns the result of that subrequest (as bytes).
+
+    The body must be JSON containing two always-required keys:
+
+    - "endpoint" -- the HTTP endpoint to invoke (e.g. "/room/some-room").
+    - "method" -- the HTTP method (e.g. "POST", "GET")
+
+    Plus, when method is POST or PUT, the required field:
+
+    - "body" -- the request body for POST/PUT requests
+
+    Optional keys that may be included are:
+    - "headers" -- optional dict of HTTP headers for the request.  Header names are
+                   case-insensitive (i.e. `X-Foo` and `x-FoO` are equivalent).
+
+    When returning, we invoke the subrequest and then, if it returns a 200 response code, we take
+    the response body, encrypt it, and then base64 the encrypted body and send that back as the
+    response body of the onion request.
+
+    If the subrequest returned a non-200 response code then instead of the returned body we return
+    `{"status_code":xxx}` (where xxx is the numeric status code) and encrypt/base64 encode that.
+
+    Response headers are completely ignored, as are bodies of non-200 responses.
+
+    This is deprecated because it amplifies request and response sizes, it doesn't allow non-json
+    requests, and it drops pertinent request information (such as response headers and error
+    bodies).  Prefer v4 requests which do not have these drawbacks.
     """
 
-    try:
-        junk = onionparser.parse_junk(request.data)
-    except RuntimeError as e:
-        app.logger.warn("Failed to decrypt onion request: {}".format(e))
-        return Response(status=http.INTERNAL_SERVER_ERROR)
+    junk = decrypt_onionreq()
+    return utils.encode_base64(junk.transformReply(handle_v3_onionreq_plaintext(junk.payload)))
 
-    response = handle_onionreq_plaintext(junk.payload)
-    return base64.b64encode(junk.transformReply(response)).decode()
+
+@app.post("/oxen/v4/lsrpc")
+def handle_v4_onion_request():
+    """
+    Handles a decrypted v4 onion request; this injects a subrequest to process it then returns the
+    result of that subrequest.  In contrast to v3, it is more efficient (particularly for binary
+    input or output) and allows using endpoints that return headers or bodies with non-2xx response
+    codes.
+
+    The body of a v4 request (post-decryption) is a bencoded list containing exactly 1 or 2 byte
+    strings: the first byte string contains a json object containing the request metadata which has
+    three required fields:
+
+    - "endpoint" -- the HTTP endpoint to invoke (e.g. "/room/some-room").
+    - "method" -- the HTTP method (e.g. "POST", "GET")
+    - "headers" -- dict of HTTP headers for the request.  Header names are case-insensitive (i.e.
+      `X-Foo` and `x-FoO` are equivalent).
+
+    Unlike v3 requests, endpoints must always start with a /.  (If a legacy endpoint "whatever"
+    needs to be accessed through a v4 request for some reason then it can be accessed via the
+    "/legacy/whatever" endpoint).
+
+    The "headers" field typically carries X-SOGS-* authentication headers as well as fields like
+    Content-Type.  Note that, unlike v3 requests, the Content-Type does *not* have any default and
+    should also be specified, often as `application/json`.  Unlike HTTP requests, Content-Length is
+    not required and will be ignored if specified; the content-length is always determined from the
+    provided body.
+
+    The second byte string in the request, if present, is the request body in raw bytes and is
+    required for POST and PUT requests and must not be provided for GET/DELETE requests.
+
+    Bencoding details:
+        A full bencode library can be used, but the format used here is deliberately meant to be as
+        simple as possible to implement without a full bencode library on hand.  The format of a
+        byte string is `N:` where N is a decimal number (e.g. `123:` starts a 123-byte string),
+        followed by the N bytes.  A list of strings starts with `l`, contains any number of encoded
+        byte strings, followed by `e`.  (Full bencode allows dicts, integers, and list/dict
+        recursion, but we do not use any of that for v4 bencoded onion requests).
+
+    For example, the request:
+
+        GET /room/some-room
+        Some-Header: 12345
+
+    would be encoded as:
+
+        l79:{"method":"GET","endpoint":"/room/some-room","headers":{"Some-Header":"12345"}}e
+
+    that is: a list containing a single 79-byte string.  A POST request such as:
+
+        POST /some/thing
+        Some-Header: a
+
+        post body here
+
+    would be encoded as the two-string bencoded list:
+
+        l72:{"method":"POST","endpoint":"/some/thing","headers":{"Some-Header":"a"}}14:post body heree
+            ^^^^^^^^72-byte request info json^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^   ^^^^^body^^^^^
+
+    The return value of the request is always a 2-part bencoded list where the first part contains
+    response metadata and the second contains the response body.  The response metadata is a json
+    object containing:
+    - "code" -- the numeric HTTP response code (e.g. 200, 403); and
+    - "headers" -- a json object of header names to values.  Note that, since HTTP headers are
+      case-insensitive, the header names are always returned as lower-case, and we strip out the
+      'content-length' header (since it is already encoded in the length of the body part).
+
+    For example, a simple json request response might be the two parts:
+
+    - `{"code":200,"headers":{"content-type":"application/json"}}`
+    - `{"id": 123}`
+
+    encoded as:
+
+        l58:{"code":200,"headers":{"content-type":"application/json"}}11:{"id": 123}e
+
+    A more complicated request, for example for a file download, might return binary content such as:
+
+    - `{"code":200,"headers":{"content-type":"application/octet-stream","content-disposition":"attachment; filename*=UTF-8''filename.txt"}}`
+    - `My file contents`
+
+    i.e. encoded as `l132:{...the json above...}16:My file contentse`
+
+    Error responses (e.g. a 403) are not treated specially; that is: they still have a "code" set to
+    the response code and "headers" and a body part of whatever the request returned for a body).
+
+    The final value returned from the endpoint is the encrypted bencoded bytes, and these encrypted
+    bytes are returned directly to the client (i.e. no base64 encoding applied, unlike v3 requests).
+    """  # noqa: E501
+
+    # Some less-than-ideal decisions in the onion request protocol design means that we are stuck
+    # dealing with parsing the request body here in the internal format that is meant for storage
+    # server, but the *last* hop's decrypted, encoded data has to get shared by us (and is passed on
+    # to us in its raw, encoded form).  It looks like this:
+    #
+    # [N][blob][json]
+    #
+    # where N is the size of blob (4 bytes, little endian), and json contains *both* the elements
+    # that were meant for the last hop (like our host/port/protocol) *and* the elements that *we*
+    # need to decrypt blob (specifically: "ephemeral_key" and, optionally, "enc_type" [which can be
+    # used to use xchacha20-poly1305 encryption instead of AES-GCM]).
+    #
+    # The parse_junk here takes care of decoding and decrypting this according to the fields *meant
+    # for us* in the json (which include things like the encryption type and ephemeral key):
+    try:
+        junk = crypto.parse_junk(request.data)
+    except RuntimeError as e:
+        app.logger.warning("Failed to decrypt onion request: {}".format(e))
+        abort(http.BAD_REQUEST)
+
+    # On the way back out we re-encrypt via the junk parser (which uses the ephemeral key and
+    # enc_type that were specified in the outer request).  We then return that encrypted binary
+    # payload as-is back to the client which bounces its way through the SN path back to the client.
+    response = handle_v4_onionreq_plaintext(junk.payload)
+    return junk.transformReply(response)

fileserver/postfork.py

@@ -6,10 +6,11 @@ except ModuleNotFoundError:
         """Simple non-uwsgi stub that just calls the postfork function"""
 
         def __init__(self, f):
-            f()
+            self.f = f
+            self.f()
 
-        def __call__(self, f):
-            pass
+        def __call__(self):
+            self.f()
 
 
 else:

fileserver/routes.py

@@ -1,12 +1,12 @@
 from . import config
 from .web import app
-from .db import psql
-from . import http
+from . import db
+from . import http, utils
 
 import flask
 from flask import request
 import secrets
-import base64
+from base64 import urlsafe_b64encode
 from hashlib import blake2b
 import json
 import psycopg
@@ -38,7 +38,7 @@ def generate_file_id(data):
     base64 chars.  (Ideally would be 32, but that would result in base64 padding, so increased to 33
     to fit perfectly).
     """
-    return base64.urlsafe_b64encode(
+    return urlsafe_b64encode(
         blake2b(data, digest_size=33, salt=b"SessionFileSvr\0\0").digest()
     ).decode()
 
@@ -66,7 +66,7 @@ def submit_file(*, body=None, deprecated=False):
                 if not deprecated:
                     id = str(id)  # New ids are always strings; legacy requests require an integer
                 try:
-                    with psql.cursor() as cur:
+                    with db.psql.cursor() as cur:
                         cur.execute(
                             "INSERT INTO files (id, data, expiry) VALUES (%s, %s, NOW() + %s)",
                             (id, body, config.FILE_EXPIRY),
@@ -83,11 +83,11 @@ def submit_file(*, body=None, deprecated=False):
                 return error_resp(http.INSUFFICIENT_STORAGE)
 
         else:
-            with psql.transaction(), psql.cursor() as cur:
+            with db.psql.transaction(), db.psql.cursor() as cur:
                 id = generate_file_id(body)
                 try:
                     # Don't pass the data yet because we might be de-duplicating
-                    with psql.transaction():
+                    with db.psql.transaction():
                         cur.execute(
                             "INSERT INTO files (id, data, expiry) VALUES (%s, '', NOW() + %s)",
                             (id, config.FILE_EXPIRY),
@@ -125,18 +125,14 @@ def submit_file_old():
         )
         return error_resp(http.PAYLOAD_TOO_LARGE)
 
-    # base64.b64decode is picky about padding (but not, by default, about random non-alphabet
-    # characters in the middle of the data, wtf!)
-    while len(body) % 4 != 0:
-        body += "="
-    body = base64.b64decode(body, validate=True)
+    body = utils.decode_base64(body)
 
     return submit_file(body=body, deprecated=True)
 
 
-@app.route("/file/<id>")
+@app.get("/file/<id>")
 def get_file(id):
-    with psql.cursor() as cur:
+    with db.psql.cursor() as cur:
         cur.execute("SELECT data FROM files WHERE id = %s", (id,), binary=True)
         row = cur.fetchone()
         if row:
@@ -148,21 +144,21 @@ def get_file(id):
             return error_resp(http.NOT_FOUND)
 
 
-@app.route("/files/<id>")
+@app.get("/files/<id>")
 def get_file_old(id):
-    with psql.cursor() as cur:
+    with db.psql.cursor() as cur:
         cur.execute("SELECT data FROM files WHERE id = %s", (id,), binary=True)
         row = cur.fetchone()
         if row:
-            return json_resp({"status_code": 200, "result": base64.b64encode(row[0]).decode()})
+            return json_resp({"status_code": 200, "result": utils.encode_base64(row[0])})
         else:
             app.logger.warn("File '{}' does not exist".format(id))
             return error_resp(http.NOT_FOUND)
 
 
-@app.route("/file/<id>/info")
+@app.get("/file/<id>/info")
 def get_file_info(id):
-    with psql.cursor() as cur:
+    with db.psql.cursor() as cur:
         cur.execute("SELECT length(data), uploaded, expiry FROM files WHERE id = %s", (id,))
         row = cur.fetchone()
         if row:
@@ -174,16 +170,16 @@ def get_file_info(id):
             return error_resp(http.NOT_FOUND)
 
 
-@app.route("/session_version")
+@app.get("/session_version")
 def get_session_version():
-    platform = request.args["platform"]
+    platform = request.args.get("platform")
 
     if platform not in ("desktop", "android", "ios"):
         app.logger.warn("Invalid session platform '{}'".format(platform))
         return error_resp(http.NOT_FOUND)
     project = "oxen-io/session-" + platform
 
-    with psql.cursor() as cur:
+    with db.psql.cursor() as cur:
         cur.execute(
             """
             SELECT version, updated FROM release_versions

fileserver/subrequest.py

@@ -0,0 +1,95 @@
+from .web import app
+from . import http
+
+from flask import request
+from io import BytesIO
+import traceback
+from typing import Optional, Union
+
+
+def make_subrequest(
+    method: str,
+    path: str,
+    *,
+    headers={},
+    content_type: Optional[str] = None,
+    body: Optional[Union[bytes, memoryview]] = None,
+    json: Optional[Union[dict, list]] = None,
+):
+    """
+    Makes a subrequest from the given parameters, returns the response object and a dict of
+    lower-case response headers keys to header values.
+
+    Parameters:
+    method - the HTTP method, e.g. GET or POST
+    path - the request path (optionally including a query string)
+    headers - dict of HTTP headers for the request
+    content_type - the content-type of the request (for POST/PUT methods)
+    body - the bytes content of the body of a POST/PUT method.  If specified then content_type will
+    default to 'application/octet-stream'.
+    json - a json value to dump as the body of the request.  If specified then content_type will
+    default to 'applicaton/json'.
+    """
+
+    http_headers = {'HTTP_{}'.format(h.upper().replace('-', '_')): v for h, v in headers.items()}
+
+    if content_type is None:
+        if 'HTTP_CONTENT_TYPE' in http_headers:
+            content_type = http_headers['HTTP_CONTENT_TYPE']
+        elif body is not None:
+            content_type = 'application/octet-stream'
+        elif json is not None:
+            content_type = 'application/json'
+        else:
+            content_type = ''
+
+    for x in ('HTTP_CONTENT_TYPE', 'HTTP_CONTENT_LENGTH'):
+        if x in http_headers:
+            del http_headers[x]
+
+    if body is None:
+        if json is not None:
+            from json import dumps
+
+            body = dumps(json, separators=(',', ':')).encode()
+        else:
+            body = b''
+
+    body_input = BytesIO(body)
+    content_length = len(body)
+
+    if '?' in path:
+        path, query_string = path.split('?', 1)
+    else:
+        query_string = ''
+
+    # Set up the wsgi environ variables for the subrequest (see PEP 0333)
+    subreq_env = {
+        **request.environ,
+        "REQUEST_METHOD": method,
+        "PATH_INFO": path,
+        "QUERY_STRING": query_string,
+        "CONTENT_TYPE": content_type,
+        "CONTENT_LENGTH": content_length,
+        **http_headers,
+        'wsgi.input': body_input,
+        'flask._preserve_context': False,
+    }
+
+    try:
+        app.logger.debug(f"Initiating sub-request for {method} {path}")
+        with app.request_context(subreq_env):
+            response = app.full_dispatch_request()
+        if response.status_code != http.OK:
+            app.logger.warning(
+                f"Sub-request for {method} {path} returned status {response.status_code}"
+            )
+        return response, {
+            k.lower(): v
+            for k, v in response.get_wsgi_headers(subreq_env)
+            if k.lower() != 'content-length'
+        }
+
+    except Exception:
+        app.logger.warning(f"Sub-request for {method} {path} failed: {traceback.format_exc()}")
+        raise

fileserver/utils.py

@@ -0,0 +1,35 @@
+from typing import Tuple
+import base64
+
+
+def bencode_consume_string(body: memoryview) -> Tuple[memoryview, memoryview]:
+    """
+    Parses a bencoded byte string from the beginning of `body`.  Returns a pair of memoryviews on
+    success: the first is the string byte data; the second is the remaining data (i.e. after the
+    consumed string).
+    Raises ValueError on parse failure.
+    """
+    pos = 0
+    print(f"body: {body.tobytes()}")
+    while pos < len(body) and 0x30 <= body[pos] <= 0x39:  # 1+ digits
+        pos += 1
+    if pos == 0 or pos >= len(body) or body[pos] != 0x3A:  # 0x3a == ':'
+        raise ValueError("Invalid string bencoding: did not find `N:` length prefix")
+
+    strlen = int(body[0:pos])  # parse the digits as a base-10 integer
+    pos += 1  # skip the colon
+    if pos + strlen > len(body):
+        raise ValueError("Invalid string bencoding: length exceeds buffer")
+    return body[pos : pos + strlen], body[pos + strlen :]
+
+
+def encode_base64(data: bytes):
+    return base64.b64encode(data).decode()
+
+
+def decode_base64(b64: str):
+    """Decodes a base64 value with or without padding."""
+    # Accept unpadded base64 by appending padding; b64decode won't accept it otherwise
+    if 2 <= len(b64) % 4 <= 3 and not b64.endswith('='):
+        b64 += '=' * (4 - len(b64) % 4)
+    return base64.b64decode(b64, validate=True)

libonionrequests

@@ -1 +0,0 @@
-Subproject commit bc373fcb0915e0378a898762e239266409d62f0c

tests/conftest.py

@@ -0,0 +1,71 @@
+import pytest
+import os
+
+from fileserver import config
+
+config.pgsql_connect_opts = {"defer": True}
+
+from fileserver import web  # noqa: E402
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--pgsql",
+        type=str,
+        help='Use the given postgresql database connect string for testing.  '
+        'E.g. "dbname=test user=joe" or "postgresql://..."',
+        required=True,
+    )
+    parser.addoption(
+        "--no-drop-schema",
+        action="store_true",
+        default=False,
+        help="Don't clean up the final test schema; typically used with --maxfail=1",
+    )
+
+
+@pytest.fixture(scope="session")
+def db_conn(request):
+    from fileserver import db as db_
+
+    pgsql = request.config.getoption("--pgsql")
+    web.app.logger.warning(f"using postgresql {pgsql}")
+
+    config.pgsql_connect_opts = {"conninfo": pgsql}
+    db_.pg_connect()
+    db_.psql = db_.psql_pool.getconn()
+
+    yield db_.psql
+
+    web.app.logger.warning("closing db")
+    if not request.config.getoption("--no-drop-schema"):
+        web.app.logger.warning("DROPPING SCHEMA")
+        with db_.psql.cursor() as cur:
+            cur.execute("DROP SCHEMA sfs_tests CASCADE")
+
+
+@pytest.fixture(autouse=True)
+def db(request, db_conn):
+    """
+    Import this fixture to get a wiped, re-initialized database for db.engine.  The actual fixture
+    value is the db module itself (so typically you don't import it at all but instead get it
+    through this fixture, which also creates an empty db for you).
+    """
+
+    with db_conn.transaction(), db_conn.cursor() as cur, open(
+        os.path.dirname(__file__) + "/../schema.pgsql", "r"
+    ) as schema:
+        cur.execute("DROP SCHEMA IF EXISTS sfs_tests CASCADE")
+        cur.execute("CREATE SCHEMA IF NOT EXISTS sfs_tests")
+        cur.execute("SET search_path TO sfs_tests")
+        cur.execute(schema.read())
+
+    return db_conn
+
+
+@pytest.fixture
+def client():
+    """Yields an flask test client for the app that can be used to make test requests"""
+
+    with web.app.test_client() as client:
+        yield client

tests/test_onion_requests.py

@@ -0,0 +1,241 @@
+from fileserver.web import app
+from fileserver import crypto, db, utils
+from nacl.bindings import (
+    crypto_scalarmult,
+    crypto_aead_xchacha20poly1305_ietf_encrypt,
+    crypto_aead_xchacha20poly1305_ietf_decrypt,
+)
+from Cryptodome.Cipher import AES
+import nacl.utils
+import nacl.hashlib
+import struct
+import json
+import time
+
+from nacl.public import PrivateKey
+
+# ephemeral X25519 keypair for use in tests
+a = PrivateKey(bytes.fromhex('ec32fb5766cf52b1b5d7b0bff08e29f5c0c58ca19beaf6a5c7d3dd8ac7ced963'))
+A = a.public_key
+assert A.encode().hex() == 'd79a50b82ba8ca665f854382b42ba159efd16eef87409e97a8d07395b9492928'
+
+# For xchacha20 we use the libsodium recommended shared key of H(aB || A || B), where H(.) is
+# 32-byte Blake2B
+shared_xchacha20_key = nacl.hashlib.blake2b(
+    crypto_scalarmult(a.encode(), crypto.privkey.public_key.encode())
+    + A.encode()
+    + crypto.privkey.public_key.encode(),
+    digest_size=32,
+).digest()
+
+# AES-GCM onion requests were implemented using the somewhat weaker shared key of just aB:
+shared_aes_key = crypto_scalarmult(a.encode(), crypto.privkey.public_key.encode())
+
+
+def build_payload(inner_json, inner_body=None, *, v, enc_type, outer_json_extra={}):
+    """Encrypt and encode a payload for fileserver"""
+
+    if not isinstance(inner_json, bytes):
+        inner_json = json.dumps(inner_json).encode()
+
+    if v == 3:
+        assert inner_body is None
+        inner_data = inner_json
+    elif v == 4:
+        inner_data = b''.join(
+            (
+                b'l',
+                str(len(inner_json)).encode(),
+                b':',
+                inner_json,
+                *(() if inner_body is None else (str(len(inner_body)).encode(), b':', inner_body)),
+                b'e',
+            )
+        )
+    else:
+        raise RuntimeError(f"invalid payload v{v}")
+
+    inner_enc = ()
+    if enc_type in ("xchacha20", "xchacha20-poly1305"):
+        # For xchacha20 we stick the nonce to the beginning of the encrypted blob
+        nonce = nacl.utils.random(24)
+        inner_enc = (
+            nonce,
+            crypto_aead_xchacha20poly1305_ietf_encrypt(
+                inner_data, aad=None, nonce=nonce, key=shared_xchacha20_key
+            ),
+        )
+    elif enc_type in ("aes-gcm", "gcm"):
+        # For aes-gcm we stick the iv on the beginning of the encrypted blob and the mac tag on the
+        # end of it
+        iv = nacl.utils.random(12)
+        cipher = AES.new(shared_aes_key, AES.MODE_GCM, iv)
+        enc, mac = cipher.encrypt_and_digest(inner_data)
+        inner_enc = (iv, enc, mac)
+    else:
+        raise ValueError(f"Invalid enc_type: {enc_type}")
+
+    # The outer request is in storage server onion request format:
+    # [N][junk]{json}
+    # where we load the fields for the last hop *and* the fields for fileserver into the json.
+    outer_json = {
+        "host": "localhost",
+        "port": 80,
+        "protocol": "http",
+        "target": f"/oxen/v{v}/lsrpc",
+        "ephemeral_key": A.encode().hex(),
+        "enc_type": enc_type,
+        **outer_json_extra,
+    }
+    return b''.join(
+        (
+            struct.pack('<i', sum(len(x) for x in inner_enc)),
+            *inner_enc,
+            json.dumps(outer_json).encode(),
+        )
+    )
+
+
+def decrypt_reply(data, *, v, enc_type):
+    """
+    Parses a reply; returns the json metadata and the body.  Note for v3 that there is only json;
+    body will always be None.
+    """
+    if v == 3:
+        data = utils.decode_base64(data)
+
+    if enc_type in ("xchacha20", "xchacha20-poly1305"):
+        assert len(data) > 24
+        nonce, enc = data[:24], data[24:]
+        data = crypto_aead_xchacha20poly1305_ietf_decrypt(
+            enc, aad=None, nonce=nonce, key=shared_xchacha20_key
+        )
+    elif enc_type in ("aes-gcm", "gcm"):
+        assert len(data) > 28
+        iv, enc, mac = data[:12], data[12:-16], data[-16:]
+        cipher = AES.new(shared_aes_key, AES.MODE_GCM, iv)
+        data = cipher.decrypt_and_verify(enc, mac)
+    else:
+        raise ValueError(f"Invalid enc_type: {enc_type}")
+
+    body = None
+
+    if v == 4:
+        assert (data[:1], data[-1:]) == (b'l', b'e')
+        data = memoryview(data)[1:-1]
+        json_data, data = utils.bencode_consume_string(data)
+        json_ = json.loads(json_data.tobytes())
+        if data:
+            body, data = utils.bencode_consume_string(data)
+            assert len(data) == 0
+            body = body.tobytes()
+    elif v == 3:
+        json_ = json.loads(data)
+
+    return json_, body
+
+
+def update_session_desktop_version():
+    """
+    Fileserver errors if the db update version is more than 24 hours old, so update it to a fake
+    version for testing.
+    """
+
+    with db.psql.cursor() as cur:
+        cur.execute(
+            "UPDATE release_versions SET version = %s, updated = NOW() WHERE project = %s",
+            ('v1.2.3', 'oxen-io/session-desktop'),
+        )
+
+
+def test_v3(client):
+    update_session_desktop_version()
+
+    # Construct an onion request for /room/test-room
+    req = {'method': 'GET', 'endpoint': 'session_version?platform=desktop'}
+    data = build_payload(req, v=3, enc_type="xchacha20")
+
+    r = client.post("/loki/v3/lsrpc", data=data)
+
+    assert r.status_code == 200
+
+    v = decrypt_reply(r.data, v=3, enc_type="xchacha20")[0]
+
+    assert -1 < time.time() - v.pop('updated') < 1
+    assert v == {'status_code': 200, 'result': 'v1.2.3'}
+
+
+def test_v4(client):
+    update_session_desktop_version()
+
+    req = {'method': 'GET', 'endpoint': '/session_version?platform=desktop'}
+    data = build_payload(req, v=4, enc_type="xchacha20")
+
+    r = client.post("/oxen/v4/lsrpc", data=data)
+
+    assert r.status_code == 200
+
+    info, body = decrypt_reply(r.data, v=4, enc_type="xchacha20")
+
+    assert info == {'code': 200, 'headers': {'content-type': 'application/json'}}
+
+    v = json.loads(body)
+    assert -1 < time.time() - v.pop('updated') < 1
+    assert v == {'status_code': 200, 'result': 'v1.2.3'}
+
+
+@app.post("/test_v4_post_body")
+def v4_post_body():
+    from flask import request, jsonify, Response
+
+    if request.json is not None:
+        return jsonify({"json": request.json})
+    print(f"rd: {request.data}")
+    return Response(
+        f"not json ({request.content_type}): {request.data.decode()}".encode(),
+        mimetype='text/plain',
+    )
+
+
+def test_v4_post_body(client):
+    req = {'method': 'POST', 'endpoint': '/test_v4_post_body'}
+    content = b'test data'
+    req['headers'] = {'content-type': 'text/plain'}
+
+    data = build_payload(req, content, v=4, enc_type="xchacha20")
+
+    r = client.post("/oxen/v4/lsrpc", data=data)
+
+    assert r.status_code == 200
+
+    info, body = decrypt_reply(r.data, v=4, enc_type="xchacha20")
+
+    assert info == {'code': 200, 'headers': {'content-type': 'text/plain; charset=utf-8'}}
+    assert body == b'not json (text/plain): test data'
+
+    # Now try with json:
+    test_json = {"test": ["json", None], "1": 23}
+    content = json.dumps(test_json).encode()
+    req['headers'] = {'content-type': 'application/json'}
+
+    data = build_payload(req, content, v=4, enc_type="xchacha20")
+    r = client.post("/oxen/v4/lsrpc", data=data)
+
+    assert r.status_code == 200
+
+    info, body = decrypt_reply(r.data, v=4, enc_type="xchacha20")
+
+    assert info == {'code': 200, 'headers': {'content-type': 'application/json'}}
+    assert json.loads(body) == {"json": test_json}
+
+    # Now try with json, but with content-type set to something else (this should avoid the json
+    req['headers'] = {'content-type': 'x-omg/all-your-base'}
+    data = build_payload(req, content, v=4, enc_type="xchacha20")
+    r = client.post("/oxen/v4/lsrpc", data=data)
+
+    assert r.status_code == 200
+
+    info, body = decrypt_reply(r.data, v=4, enc_type="xchacha20")
+
+    assert info == {'code': 200, 'headers': {'content-type': 'text/plain; charset=utf-8'}}
+    assert body == b'not json (x-omg/all-your-base): ' + content