Merge pull request #446 from jagerman/storage-tags

Storage namespaces & authentication

.clang-format

@@ -14,7 +14,7 @@ AllowShortIfStatementsOnASingleLine: 'false'
 AllowShortLoopsOnASingleLine: 'false'
 AlwaysBreakAfterReturnType: None
 AlwaysBreakTemplateDeclarations: Yes
-BreakBeforeBinaryOperators: NonAssignment
+BreakBeforeBinaryOperators: None
 BreakBeforeBraces: Attach
 BreakBeforeTernaryOperators: 'true'
 BreakConstructorInitializers: AfterColon

.gitmodules

@@ -19,9 +19,6 @@
 [submodule "vendors/SQLiteCpp"]
 	path = vendors/SQLiteCpp
 	url = https://github.com/SRombauts/SQLiteCpp.git
-[submodule "network-tests/pyoxenmq"]
-	path = network-tests/pyoxenmq
-	url = https://github.com/oxen-io/oxen-pyoxenmq.git
 [submodule "vendors/oxenc"]
 	path = vendors/oxenc
 	url = https://github.com/oxen-io/oxen-encoding.git

common/include/oxen_common.h

@@ -4,6 +4,7 @@
 #include <cstdint>
 #include <string>
 #include <string_view>
+#include <type_traits>
 
 namespace oxen {
 
@@ -65,10 +66,33 @@ class user_pubkey_t {
     std::string prefixed_raw() const;
 };
 
+enum class namespace_id : int16_t {
+    Default = 0,  // Ordinary Session messages
+    Min = -32768,
+    Max = 32767,
+    SessionSync = 5,     // Session sync data for imports & multidevice syncing
+    ClosedV2 = 3,        // Reserved for future Session closed group implementations
+    LegacyClosed = -10,  // For "old" closed group messages; allows unauthenticated retrieval
+};
+
+constexpr bool is_public_namespace(namespace_id ns) {
+    return static_cast<std::underlying_type_t<namespace_id>>(ns) % 10 == 0;
+}
+
+constexpr auto to_int(namespace_id ns) {
+    return static_cast<std::underlying_type_t<namespace_id>>(ns);
+}
+
+std::string to_string(namespace_id ns);
+
+constexpr auto NAMESPACE_MIN = to_int(namespace_id::Min);
+constexpr auto NAMESPACE_MAX = to_int(namespace_id::Max);
+
 /// message received from a client
 struct message {
     user_pubkey_t pubkey;
     std::string hash;
+    namespace_id msg_namespace;
     std::chrono::system_clock::time_point timestamp;
     std::chrono::system_clock::time_point expiry;
     std::string data;
@@ -77,20 +101,27 @@ struct message {
 
     message(user_pubkey_t pubkey,
             std::string hash,
+            namespace_id msg_ns,
             std::chrono::system_clock::time_point timestamp,
             std::chrono::system_clock::time_point expiry,
             std::string data) :
             pubkey{std::move(pubkey)},
             hash{std::move(hash)},
+            msg_namespace{msg_ns},
             timestamp{timestamp},
             expiry{expiry},
             data{std::move(data)} {}
 
     message(std::string hash,
+            namespace_id msg_ns,
             std::chrono::system_clock::time_point timestamp,
             std::chrono::system_clock::time_point expiry,
             std::string data) :
-            hash{std::move(hash)}, timestamp{timestamp}, expiry{expiry}, data{std::move(data)} {}
+            hash{std::move(hash)},
+            msg_namespace{msg_ns},
+            timestamp{timestamp},
+            expiry{expiry},
+            data{std::move(data)} {}
 };
 
 using swarm_id_t = uint64_t;

common/src/oxen_common.cpp

@@ -1,8 +1,18 @@
 #include "oxen_common.h"
 #include <oxenc/hex.h>
+#include <charconv>
+#include <cassert>
 
 namespace oxen {
 
+std::string to_string(namespace_id ns) {
+    char buf[6];
+    static_assert(NAMESPACE_MIN >= -99'999 && NAMESPACE_MAX <= 999'999);
+    auto [ptr, ec] = std::to_chars(std::begin(buf), std::end(buf), to_int(ns));
+    assert(ec == std::errc());
+    return std::string(std::begin(buf), ptr - std::begin(buf));
+}
+
 user_pubkey_t& user_pubkey_t::load(std::string_view pk) {
     if (pk.size() == USER_PUBKEY_SIZE_HEX && oxenc::is_hex(pk)) {
         uint8_t netid;

crypto/src/channel_encryption.cpp

@@ -174,8 +174,8 @@ static std::string decrypt_openssl(
     }
     o += len;
 
-    if (!tag.empty()
-        && EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, taglen, (void*)tag.data()) <= 0)
+    if (!tag.empty() &&
+        EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, taglen, (void*)tag.data()) <= 0)
         throw std::runtime_error{"Could not set decryption tag"};
 
     // Decrypt any remaining partial blocks
@@ -233,11 +233,10 @@ static std::array<unsigned char, crypto_aead_xchacha20poly1305_ietf_KEYBYTES> xc
         bool local_first) {
     std::array<unsigned char, crypto_aead_xchacha20poly1305_ietf_KEYBYTES> key;
     static_assert(crypto_aead_xchacha20poly1305_ietf_KEYBYTES >= crypto_scalarmult_BYTES);
-    if (0
-        != crypto_scalarmult(
-                key.data(),
-                local_sec.data(),
-                remote_pub.data()))  // Use key as tmp storage for aB
+    if (0 != crypto_scalarmult(
+                     key.data(),
+                     local_sec.data(),
+                     remote_pub.data()))  // Use key as tmp storage for aB
         throw std::runtime_error{"Failed to compute shared key for xchacha20"};
     crypto_generichash_state h;
     crypto_generichash_init(&h, nullptr, 0, key.size());
@@ -254,16 +253,16 @@ std::string ChannelEncryption::encrypt_xchacha20(
 
     std::string ciphertext;
     ciphertext.resize(
-            crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + plaintext.size()
-            + crypto_aead_xchacha20poly1305_ietf_ABYTES);
+            crypto_aead_xchacha20poly1305_ietf_NPUBBYTES + plaintext.size() +
+            crypto_aead_xchacha20poly1305_ietf_ABYTES);
 
     const auto key = xchacha20_shared_key(public_key_, private_key_, pubKey, !server_);
 
     // Generate random nonce, and stash it at the beginning of ciphertext:
     randombytes_buf(ciphertext.data(), crypto_aead_xchacha20poly1305_ietf_NPUBBYTES);
 
-    auto* c = reinterpret_cast<unsigned char*>(ciphertext.data())
-            + crypto_aead_xchacha20poly1305_ietf_NPUBBYTES;
+    auto* c = reinterpret_cast<unsigned char*>(ciphertext.data()) +
+              crypto_aead_xchacha20poly1305_ietf_NPUBBYTES;
     unsigned long long clen;
 
     crypto_aead_xchacha20poly1305_ietf_encrypt(
@@ -297,17 +296,16 @@ std::string ChannelEncryption::decrypt_xchacha20(
     plaintext.resize(ciphertext.size() - crypto_aead_xchacha20poly1305_ietf_ABYTES);
     auto* m = reinterpret_cast<unsigned char*>(plaintext.data());
     unsigned long long mlen;
-    if (0
-        != crypto_aead_xchacha20poly1305_ietf_decrypt(
-                m,
-                &mlen,
-                nullptr,  // nsec (always unused)
-                ciphertext.data(),
-                ciphertext.size(),
-                nullptr,
-                0,  // additional data
-                nonce.data(),
-                key.data()))
+    if (0 != crypto_aead_xchacha20poly1305_ietf_decrypt(
+                     m,
+                     &mlen,
+                     nullptr,  // nsec (always unused)
+                     ciphertext.data(),
+                     ciphertext.size(),
+                     nullptr,
+                     0,  // additional data
+                     nonce.data(),
+                     key.data()))
         throw std::runtime_error{"Could not decrypt (XChaCha20-Poly1305)"};
     assert(mlen <= plaintext.size());
     plaintext.resize(mlen);

crypto/src/oxend_key.cpp

@@ -18,16 +18,16 @@ namespace detail {
             throw std::runtime_error{"Hex key data is invalid: data is not hex"};
         if (hex.size() != 2 * length)
             throw std::runtime_error{
-                    "Hex key data is invalid: expected " + std::to_string(length)
-                    + " hex digits, received " + std::to_string(hex.size())};
+                    "Hex key data is invalid: expected " + std::to_string(length) +
+                    " hex digits, received " + std::to_string(hex.size())};
         oxenc::from_hex(hex.begin(), hex.end(), reinterpret_cast<unsigned char*>(buffer));
     }
 
     void load_from_bytes(void* buffer, size_t length, std::string_view bytes) {
         if (bytes.size() != length)
             throw std::runtime_error{
-                    "Key data is invalid: expected " + std::to_string(length) + " bytes, received "
-                    + std::to_string(bytes.size())};
+                    "Key data is invalid: expected " + std::to_string(length) +
+                    " bytes, received " + std::to_string(bytes.size())};
         std::memmove(buffer, bytes.data(), length);
     }
 
@@ -68,8 +68,8 @@ static T parse_pubkey(std::string_view pubkey_in) {
     else if (pubkey_in.size() == 64 && oxenc::is_hex(pubkey_in))
         oxenc::from_hex(pubkey_in.begin(), pubkey_in.end(), pk.begin());
     else if (
-            (pubkey_in.size() == 43 || (pubkey_in.size() == 44 && pubkey_in.back() == '='))
-            && oxenc::is_base64(pubkey_in))
+            (pubkey_in.size() == 43 || (pubkey_in.size() == 44 && pubkey_in.back() == '=')) &&
+            oxenc::is_base64(pubkey_in))
         oxenc::from_base64(pubkey_in.begin(), pubkey_in.end(), pk.begin());
     else if (pubkey_in.size() == 52 && oxenc::is_base32z(pubkey_in))
         oxenc::from_base32z(pubkey_in.begin(), pubkey_in.end(), pk.begin());

crypto/src/signature.cpp

@@ -169,8 +169,8 @@ signature signature::from_base64(std::string_view signature_b64) {
         throw std::runtime_error{"Invalid data: not base64-encoded"};
 
     // 64 bytes bytes -> 86/88 base64 encoded bytes with/without padding
-    if (!(signature_b64.size() == 86
-          || (signature_b64.size() == 88 && signature_b64.substr(86) == "==")))
+    if (!(signature_b64.size() == 86 ||
+          (signature_b64.size() == 88 && signature_b64.substr(86) == "==")))
         throw std::runtime_error{"Invalid data: b64 data size does not match signature size"};
 
     // convert signature

httpserver/client_rpc_endpoints.cpp

@@ -4,8 +4,10 @@
 #include "time.hpp"
 
 #include <chrono>
+#include <limits>
 #include <type_traits>
 #include <unordered_set>
+#include <variant>
 
 #include <oxenc/base64.h>
 #include <oxenc/hex.h>
@@ -20,35 +22,55 @@ using std::chrono::system_clock;
 
 namespace {
     template <typename T>
-    constexpr std::string_view type_desc =
-            std::is_same_v<T, bool>                       ? "boolean"sv
-            : std::is_unsigned_v<T>                       ? "positive integer"sv
-            : std::is_integral_v<T>                       ? "integer"sv
-            : std::is_same_v<T, system_clock::time_point> ? "integer timestamp (in milliseconds)"sv
-            : std::is_same_v<T, std::vector<std::string>> ? "string array"sv
-                                                          : "string"sv;
+    constexpr bool is_timestamp = std::is_same_v<T, system_clock::time_point>;
+    template <typename T>
+    constexpr bool is_str_array = std::is_same_v<T, std::vector<std::string>>;
+    template <typename T>
+    constexpr bool is_namespace_var = std::is_same_v<T, namespace_var>;
+
+    template <typename T>
+    constexpr std::string_view type_desc = std::is_same_v<T, bool>         ? "boolean"sv
+                                         : std::is_unsigned_v<T>           ? "positive integer"sv
+                                         : std::is_integral_v<T>           ? "integer"sv
+                                         : is_namespace_var<T>             ? "integer or \"all\""sv
+                                         : std::is_same_v<T, namespace_id> ? "16-bit integer"sv
+                                         : is_timestamp<T> ? "integer timestamp (in milliseconds)"sv
+                                         : is_str_array<T> ? "string array"sv
+                                                           : "string"sv;
+
+    template <typename... T>
+    constexpr bool is_monostate_var = false;
+    template <typename... T>
+    constexpr bool is_monostate_var<std::variant<std::monostate, T...>> = true;
+
+    template <typename T>
+    using maybe_type = std::conditional_t<is_monostate_var<T>, T, std::optional<T>>;
+
+    template <typename T>
+    constexpr bool is_parseable_v =
+            std::is_unsigned_v<T> || std::is_integral_v<T> || is_timestamp<T> || is_str_array<T> ||
+            is_namespace_var<T> || std::is_same_v<T, std::string_view> ||
+            std::is_same_v<T, std::string> || std::is_same_v<T, namespace_id>;
 
     // Extracts a field suitable for a `T` value from the given json with name `name`.  Takes
     // the json params and the name.  Throws if it encounters an invalid value (i.e. expecting a
     // number but given a bool).  Returns nullopt if the field isn't present or is present and
     // set to null.
     template <typename T>
-    std::optional<T> parse_field(const json& params, const char* name) {
-        constexpr bool is_timestamp = std::is_same_v<T, system_clock::time_point>;
-        constexpr bool is_str_array = std::is_same_v<T, std::vector<std::string>>;
-        static_assert(
-                std::is_unsigned_v<T> || std::is_integral_v<T> || is_timestamp || is_str_array
-                || std::is_same_v<T, std::string_view> || std::is_same_v<T, std::string>);
+    maybe_type<T> parse_field(const json& params, const char* name) {
+        static_assert(is_parseable_v<T>);
         auto it = params.find(name);
         if (it == params.end() || it->is_null())
-            return std::nullopt;
+            return {};
 
-        bool right_type = std::is_same_v<T, bool>               ? it->is_boolean()
-                        : std::is_unsigned_v<T> || is_timestamp ? it->is_number_unsigned()
-                        : std::is_integral_v<T>                 ? it->is_number_integer()
-                        : is_str_array                          ? it->is_array()
-                                                                : it->is_string();
-        if (is_str_array && right_type)
+        bool right_type = std::is_same_v<T, bool>                  ? it->is_boolean()
+                        : std::is_unsigned_v<T> || is_timestamp<T> ? it->is_number_unsigned()
+                        : std::is_integral_v<T> || std::is_same_v<T, namespace_id>
+                                ? it->is_number_integer()
+                        : is_namespace_var<T> ? it->is_number_integer() || it->is_string()
+                        : is_str_array<T>     ? it->is_array()
+                                              : it->is_string();
+        if (is_str_array<T> && right_type)
             for (auto& x : *it)
                 if (!x.is_string())
                     right_type = false;
@@ -57,7 +79,7 @@ namespace {
                     fmt::format("Invalid value given for '{}': expected {}", name, type_desc<T>)};
         if constexpr (std::is_same_v<T, std::string_view>)
             return it->template get_ref<const std::string&>();
-        else if constexpr (is_timestamp) {
+        else if constexpr (is_timestamp<T>) {
             auto time = from_epoch_ms(it->template get<int64_t>());
             // If we get a small timestamp value (less than 1M seconds since epoch) then this
             // was very likely given as unix epoch seconds rather than milliseconds
@@ -65,22 +87,32 @@ namespace {
                 throw parse_error{fmt::format(
                         "Invalid timestamp for '{}': timestamp must be in milliseconds", name)};
             return time;
-        } else
+        } else if constexpr (is_namespace_var<T> || std::is_same_v<T, namespace_id>) {
+            if (it->is_number_integer()) {
+                int64_t id = it->get<int64_t>();
+                if (id < NAMESPACE_MIN || id > NAMESPACE_MAX)
+                    throw parse_error{
+                            fmt::format("Invalid value given for '{}': value out of range", name)};
+                return namespace_id{static_cast<std::underlying_type_t<namespace_id>>(id)};
+            }
+            if constexpr (is_namespace_var<T>)
+                if (it->is_string() && it->get_ref<const std::string&>() == "all")
+                    return namespace_all;
+            throw parse_error{
+                    fmt::format("Invalid value given for '{}': expected integer or \"all\"", name)};
+        } else {
             return it->template get<T>();
+        }
     }
 
-    // Equivalent to the above, but for a bt_dict_consumer.  Note that this advances the current
-    // state of the bt_dict_consumer to just after the given field and so this *must* be called
-    // in sorted key order.
+    // Equivalent to the above, but for a bt_dict_consumer.  Note that this advances the
+    // current state of the bt_dict_consumer to just after the given field and so this
+    // *must* be called in sorted key order.
     template <typename T>
-    std::optional<T> parse_field(bt_dict_consumer& params, const char* name) {
-        constexpr bool is_timestamp = std::is_same_v<T, system_clock::time_point>;
-        constexpr bool is_str_array = std::is_same_v<T, std::vector<std::string>>;
-        static_assert(
-                std::is_unsigned_v<T> || std::is_integral_v<T> || is_timestamp || is_str_array
-                || std::is_same_v<T, std::string_view> || std::is_same_v<T, std::string>);
+    maybe_type<T> parse_field(bt_dict_consumer& params, const char* name) {
+        static_assert(is_parseable_v<T>);
         if (!params.skip_until(name))
-            return std::nullopt;
+            return {};
 
         try {
             if constexpr (std::is_same_v<T, std::string_view>)
@@ -89,13 +121,20 @@ namespace {
                 return params.consume_string();
             else if constexpr (std::is_integral_v<T>)
                 return params.consume_integer<T>();
-            else if constexpr (is_timestamp)
+            else if constexpr (is_timestamp<T>)
                 return from_epoch_ms(params.consume_integer<int64_t>());
-            else if constexpr (is_str_array) {
+            else if constexpr (is_str_array<T>) {
                 auto strs = std::make_optional<T>();
                 for (auto l = params.consume_list_consumer(); !l.is_finished();)
                     strs->push_back(l.consume_string());
                 return strs;
+            } else if constexpr (is_namespace_var<T> || std::is_same_v<T, namespace_id>) {
+                if (params.is_integer())
+                    return namespace_id{
+                            params.consume_integer<std::underlying_type_t<namespace_id>>()};
+                if constexpr (is_namespace_var<T>)
+                    if (params.is_string() && params.consume_string_view() == "all"sv)
+                        return namespace_all;
             }
         } catch (...) {
         }
@@ -103,10 +142,10 @@ namespace {
                 fmt::format("Invalid value given for '{}': expected {}", name, type_desc<T>)};
     }
 
-    // Backwards compat code for fields like ttl and timestamp that are accepted either as
-    // integer *or* stringified integer.
+    // Backwards compat code for fields like ttl and timestamp that are accepted either
+    // as integer *or* stringified integer.
     template <typename T, typename = std::enable_if_t<std::is_integral_v<T>>>
-    std::optional<T> parse_stringified(const json& params, const char* name) {
+    maybe_type<T> parse_stringified(const json& params, const char* name) {
         if (auto it = params.find(name); it != params.end() && it->is_string()) {
             if (T value; util::parse_int(it->get_ref<const std::string&>(), value))
                 return value;
@@ -125,17 +164,17 @@ namespace {
     }
 #endif
 
-    // Loads fields from a bt_dict_consumer or a json object.  Names must be specified in
-    // alphabetical order.  Throws a parse_error if the field exists but cannot be converted
-    // into a `T`.
+    // Loads fields from a bt_dict_consumer or a json object.  Names must be specified
+    // in alphabetical order.  Throws a parse_error if the field exists but cannot be
+    // converted into a `T`.
     template <
             typename... T,
             typename Dict,
             typename... Names,
             typename = std::enable_if_t<
-                    sizeof...(T) == sizeof...(Names)
-                    && (std::is_convertible_v<Names, std::string_view> && ...)>>
-    std::tuple<std::optional<T>...> load_fields(Dict& params, const Names&... names) {
+                    sizeof...(T) == sizeof...(Names) &&
+                    (std::is_convertible_v<Names, std::string_view> && ...)>>
+    std::tuple<maybe_type<T>...> load_fields(Dict& params, const Names&... names) {
         assert(check_ascending(names...));
         return {parse_field<T>(params, names)...};
     }
@@ -146,6 +185,12 @@ namespace {
             throw parse_error{fmt::format("Required field '{}' missing", name)};
     }
 
+    template <typename... T>
+    void require(std::string_view name, const std::variant<std::monostate, T...>& v) {
+        if (v.index() == 0)
+            throw parse_error{fmt::format("Required field '{}' missing", name)};
+    }
+
     template <typename T1, typename T2>
     void require_at_most_one_of(
             std::string_view first,
@@ -171,20 +216,31 @@ namespace {
                     second)};
     }
 
-    template <typename RPC, typename Dict>
-    static void load_pk_signature(
-            RPC& rpc,
-            const Dict&,
-            std::optional<std::string>& pk,
-            const std::optional<std::string_view>& pk_ed,
-            const std::optional<std::string_view>& sig) {
+    template <typename RPC>
+    void load_pk(RPC& rpc, std::optional<std::string>& pk) {
         require("pubkey", pk);
-        require("signature", sig);
         if (!rpc.pubkey.load(std::move(*pk)))
             throw parse_error{fmt::format(
                     "Pubkey must be {} hex digits ({} bytes) long",
                     USER_PUBKEY_SIZE_HEX,
                     USER_PUBKEY_SIZE_BYTES)};
+    }
+
+    template <typename T>
+    constexpr bool is_std_optional = false;
+    template <typename T>
+    constexpr bool is_std_optional<std::optional<T>> = true;
+
+    // Parses (but does not verify) a required request signature value.
+    template <typename RPC, typename Dict>
+    void load_pk_signature(
+            RPC& rpc,
+            const Dict&,
+            std::optional<std::string>& pk,
+            const std::optional<std::string_view>& pk_ed,
+            const std::optional<std::string_view>& sig) {
+        load_pk(rpc, pk);
+        require("signature", sig);
 
         if (pk_ed) {
             if (rpc.pubkey.type() != 5)
@@ -196,33 +252,64 @@ namespace {
             } else if (pk_ed->size() == 32) {
                 std::memcpy(rpc.pubkey_ed25519.emplace().data(), pk_ed->data(), pk_ed->size());
             } else {
-                throw parse_error{"Invalid pubkey_ed25519: expected 64 hex char or 32 byte pubkey"};
+                throw parse_error{
+                        "Invalid pubkey_ed25519: expected 64 hex char or 32 byte "
+                        "pubkey"};
             }
         }
 
+        unsigned char* sig_data_ptr;
+        if constexpr (is_std_optional<decltype(rpc.signature)>)
+            sig_data_ptr = rpc.signature.emplace().data();
+        else
+            sig_data_ptr = rpc.signature.data();
+
         if constexpr (std::is_same_v<json, Dict>) {
-            if (!oxenc::is_base64(*sig)
-                || !(sig->size() == 88 || (sig->size() == 86 && sig->substr(84) == "==")))
+            if (!oxenc::is_base64(*sig) ||
+                !(sig->size() == 88 || (sig->size() == 86 && sig->substr(84) == "==")))
                 throw parse_error{"invalid signature: expected base64 encoded Ed25519 signature"};
-            oxenc::from_base64(sig->begin(), sig->end(), rpc.signature.begin());
+            oxenc::from_base64(sig->begin(), sig->end(), sig_data_ptr);
         } else {
             if (sig->size() != 64)
                 throw parse_error{"invalid signature: expected 64-byte Ed25519 signature"};
-            std::memcpy(rpc.signature.data(), sig->data(), 64);
+            std::memcpy(sig_data_ptr, sig->data(), 64);
         }
-        // NB: We don't validate the signature here, we only parse input
+    }
+
+    void set_variant(bt_dict& dict, const std::string& key, const namespace_var& ns) {
+        if (auto* id = std::get_if<namespace_id>(&ns))
+            dict[key] = static_cast<std::underlying_type_t<namespace_id>>(*id);
+        else if (std::holds_alternative<namespace_all_t>(ns))
+            dict[key] = "all";
+        else
+            assert(std::holds_alternative<std::monostate>(ns));
     }
 
 }  // namespace
 
 template <typename Dict>
 static void load(store& s, Dict& d) {
-    auto [data, expiry, pubkey_alt, pubkey] =
-            load_fields<std::string_view, system_clock::time_point, std::string, std::string>(
-                    d, "data", "expiry", "pubKey", "pubkey");
+    auto [data, expiry, msg_ns, pubkey_alt, pubkey, pk_ed25519, sig_ts, sig] = load_fields<
+            std::string_view,
+            system_clock::time_point,
+            namespace_id,
+            std::string,
+            std::string,
+            std::string_view,
+            system_clock::time_point,
+            std::string_view>(
+            d,
+            "data",
+            "expiry",
+            "namespace",
+            "pubKey",
+            "pubkey",
+            "pubkey_ed25519",
+            "sig_timestamp",
+            "signature");
 
-    // timestamp and ttl are special snowflakes: for backwards compat reasons, they can be
-    // passed as strings when loading from json.
+    // timestamp and ttl are special snowflakes: for backwards compat reasons, they can
+    // be passed as strings when loading from json.
     std::optional<uint64_t> ttl;
     std::optional<system_clock::time_point> timestamp;
     if constexpr (std::is_same_v<Dict, json>) {
@@ -234,18 +321,23 @@ static void load(store& s, Dict& d) {
         ttl = parse_field<uint64_t>(d, "ttl");
     }
 
-    require_exactly_one_of("pubkey", pubkey, "pubKey", pubkey_alt, true);
-    if (!s.pubkey.load(std::move(pubkey ? *pubkey : *pubkey_alt)))
-        throw parse_error{fmt::format(
-                "Pubkey must be {} hex digits/{} bytes long",
-                USER_PUBKEY_SIZE_HEX,
-                USER_PUBKEY_SIZE_BYTES)};
-
     require("timestamp", timestamp);
     require_exactly_one_of("expiry", expiry, "ttl", ttl);
     s.timestamp = *timestamp;
     s.expiry = expiry ? *expiry : s.timestamp + std::chrono::milliseconds{*ttl};
 
+    require_exactly_one_of("pubkey", pubkey, "pubKey", pubkey_alt, true);
+    auto& pk = pubkey ? pubkey : pubkey_alt;
+
+    if (msg_ns)
+        s.msg_namespace = *msg_ns;
+
+    if (sig) {
+        load_pk_signature(s, d, pk, pk_ed25519, sig);
+        s.sig_ts = sig_ts.value_or(s.timestamp);
+    } else
+        load_pk(s, pk);
+
     require("data", data);
     if constexpr (std::is_same_v<Dict, json>) {
         // For json we require data be base64 encoded
@@ -276,18 +368,27 @@ void store::load_from(bt_dict_consumer params) {
     load(*this, params);
 }
 bt_value store::to_bt() const {
-    return bt_dict{
+    bt_dict d{
             {"pubkey", pubkey.prefixed_raw()},
             {"timestamp", to_epoch_ms(timestamp)},
             {"expiry", to_epoch_ms(expiry)},
             {"data", std::string_view{data}}};
+    if (msg_namespace != namespace_id::Default)
+        d["namespace"] = static_cast<std::underlying_type_t<namespace_id>>(msg_namespace);
+    if (signature)
+        d["signature"] = util::view_guts(*signature);
+    if (pubkey_ed25519)
+        d["pubkey_ed25519"] = std::string_view{
+                reinterpret_cast<const char*>(pubkey_ed25519->data()), pubkey_ed25519->size()};
+    return d;
 }
 
 template <typename Dict>
 static void load(retrieve& r, Dict& d) {
-    auto [lastHash, last_hash, pubKey, pubkey, pk_ed25519, sig, ts] = load_fields<
+    auto [lastHash, last_hash, msg_ns, pubKey, pubkey, pk_ed25519, sig, ts] = load_fields<
             std::string,
             std::string,
+            namespace_id,
             std::string,
             std::string,
             std::string_view,
@@ -296,6 +397,7 @@ static void load(retrieve& r, Dict& d) {
             d,
             "lastHash",
             "last_hash",
+            "namespace",
             "pubKey",
             "pubkey",
             "pubkey_ed25519",
@@ -303,19 +405,19 @@ static void load(retrieve& r, Dict& d) {
             "timestamp");
 
     require_exactly_one_of("pubkey", pubkey, "pubKey", pubKey, true);
+    auto& pk = pubkey ? pubkey : pubKey;
 
-    if (pk_ed25519 || sig || ts) {
-        load_pk_signature(r, d, pubkey ? pubkey : pubKey, pk_ed25519, sig);
+    if (pk_ed25519 || sig || ts || (msg_ns && *msg_ns != namespace_id::LegacyClosed)) {
+        load_pk_signature(r, d, pk, pk_ed25519, sig);
         r.timestamp = std::move(*ts);
         r.check_signature = true;
     } else {
-        if (!r.pubkey.load(std::move(pubkey ? *pubkey : *pubKey)))
-            throw parse_error{fmt::format(
-                    "Pubkey must be {} hex digits/{} bytes long",
-                    USER_PUBKEY_SIZE_HEX,
-                    USER_PUBKEY_SIZE_BYTES)};
+        load_pk(r, pk);
     }
 
+    if (msg_ns)
+        r.msg_namespace = *msg_ns;
+
     require_at_most_one_of("last_hash", last_hash, "lastHash", lastHash);
     if (lastHash)
         last_hash = std::move(lastHash);
@@ -338,10 +440,7 @@ void retrieve::load_from(bt_dict_consumer params) {
 }
 
 static bool is_valid_message_hash(std::string_view hash) {
-    return (hash.size() == 43 && oxenc::is_base64(hash)) ||
-           // TODO: remove this in the future, once everything has been upgraded to a SS
-           // that uses 43-byte base64 string hashes instead.
-           (hash.size() == 128 && oxenc::is_hex(hash));
+    return (hash.size() == 43 && oxenc::is_base64(hash));
 }
 
 template <typename Dict>
@@ -382,12 +481,17 @@ bt_value delete_msgs::to_bt() const {
 
 template <typename Dict>
 static void load(delete_all& da, Dict& d) {
-    auto [pubkey, pubkey_ed25519, signature, timestamp] =
-            load_fields<std::string, std::string_view, std::string_view, system_clock::time_point>(
-                    d, "pubkey", "pubkey_ed25519", "signature", "timestamp");
+    auto [msgs_ns, pubkey, pubkey_ed25519, signature, timestamp] = load_fields<
+            namespace_var,
+            std::string,
+            std::string_view,
+            std::string_view,
+            system_clock::time_point>(
+            d, "namespace", "pubkey", "pubkey_ed25519", "signature", "timestamp");
 
     load_pk_signature(da, d, pubkey, pubkey_ed25519, signature);
     require("timestamp", timestamp);
+    da.msg_namespace = std::move(msgs_ns);
     da.timestamp = std::move(*timestamp);
 }
 void delete_all::load_from(json params) {
@@ -401,6 +505,7 @@ bt_value delete_all::to_bt() const {
             {"pubkey", pubkey.prefixed_raw()},
             {"signature", util::view_guts(signature)},
             {"timestamp", to_epoch_ms(timestamp)}};
+    set_variant(ret, "namespace", msg_namespace);
     if (pubkey_ed25519)
         ret["pubkey_ed25519"] = std::string_view{
                 reinterpret_cast<const char*>(pubkey_ed25519->data()), pubkey_ed25519->size()};
@@ -409,13 +514,17 @@ bt_value delete_all::to_bt() const {
 
 template <typename Dict>
 static void load(delete_before& db, Dict& d) {
-    auto [before, pubkey, pubkey_ed25519, signature] =
-            load_fields<system_clock::time_point, std::string, std::string_view, std::string_view>(
-                    d, "before", "pubkey", "pubkey_ed25519", "signature");
+    auto [before, msgs_ns, pubkey, pubkey_ed25519, signature] = load_fields<
+            system_clock::time_point,
+            namespace_var,
+            std::string,
+            std::string_view,
+            std::string_view>(d, "before", "namespace", "pubkey", "pubkey_ed25519", "signature");
 
     load_pk_signature(db, d, pubkey, pubkey_ed25519, signature);
     require("before", before);
     db.before = std::move(*before);
+    db.msg_namespace = std::move(msgs_ns);
 }
 void delete_before::load_from(json params) {
     load(*this, params);
@@ -428,6 +537,7 @@ bt_value delete_before::to_bt() const {
             {"pubkey", pubkey.prefixed_raw()},
             {"signature", util::view_guts(signature)},
             {"before", to_epoch_ms(before)}};
+    set_variant(ret, "namespace", msg_namespace);
     if (pubkey_ed25519)
         ret["pubkey_ed25519"] = std::string_view{
                 reinterpret_cast<const char*>(pubkey_ed25519->data()), pubkey_ed25519->size()};
@@ -436,13 +546,17 @@ bt_value delete_before::to_bt() const {
 
 template <typename Dict>
 static void load(expire_all& e, Dict& d) {
-    auto [expiry, pubkey, pubkey_ed25519, signature] =
-            load_fields<system_clock::time_point, std::string, std::string_view, std::string_view>(
-                    d, "expiry", "pubkey", "pubkey_ed25519", "signature");
+    auto [expiry, msgs_ns, pubkey, pubkey_ed25519, signature] = load_fields<
+            system_clock::time_point,
+            namespace_var,
+            std::string,
+            std::string_view,
+            std::string_view>(d, "expiry", "namespace", "pubkey", "pubkey_ed25519", "signature");
 
     load_pk_signature(e, d, pubkey, pubkey_ed25519, signature);
     require("expiry", expiry);
     e.expiry = std::move(*expiry);
+    e.msg_namespace = std::move(msgs_ns);
 }
 void expire_all::load_from(json params) {
     load(*this, params);
@@ -455,6 +569,7 @@ bt_value expire_all::to_bt() const {
             {"pubkey", pubkey.prefixed_raw()},
             {"signature", util::view_guts(signature)},
             {"expiry", to_epoch_ms(expiry)}};
+    set_variant(ret, "namespace", msg_namespace);
     if (pubkey_ed25519)
         ret["pubkey_ed25519"] = std::string_view{
                 reinterpret_cast<const char*>(pubkey_ed25519->data()), pubkey_ed25519->size()};

httpserver/client_rpc_endpoints.h

@@ -10,6 +10,7 @@
 #include <string_view>
 
 #include <nlohmann/json.hpp>
+#include <variant>
 #include <oxenc/bt_serialize.h>
 
 namespace oxen::rpc {
@@ -73,31 +74,59 @@ namespace {
 
 /// Stores data in this service node and forwards it to the rest of the storage swarm.  Takes
 /// keys of:
-/// - `pubkey` (required) contains the pubkey of the recipient, encoded in hex.  Can also use
-/// the key name `pubKey` for this.
-/// - `timestamp` (required) the timestamp of the message in unix epoch milliseconds, passed as
-/// an integer.  Timestamp may not be in the future (though a few seconds tolerance is
-/// permitted).  For backwards compatibility may be passed as a stringified integer.
+/// - `pubkey` (required) contains the pubkey of the recipient, encoded in hex.  Can also use the
+///   key name `pubKey` for this.
+/// - `timestamp` (required) the timestamp of the message in unix epoch milliseconds, passed as an
+///   integer.  Timestamp may not be in the future (though a few seconds tolerance is permitted).
+///   For backwards compatibility may be passed as a stringified integer.
 /// - `ttl` (required, unless expiry given) the message's lifetime, in milliseconds, passed as a
-/// string or stringified integer, relative to the timestamp.  Timestamp+ttl must not be in the
-/// past.  For backwards compatibility may be passed as a stringified integer.
-/// - `expiry` (required, unless ttl given) the message's expiry time as a unix epoch
-/// milliseconds timestamp.  (Unlike ttl, this cannot be passed as a stringified integer).
-/// - `data` (required) the message data, encoded in base64 (for json requests).  Max data size
-/// is 76800 bytes (== 102400 in b64 encoding).  For OMQ RPC requests the value is bytes.
+///   string or stringified integer, relative to the timestamp.  Timestamp+ttl must not be in the
+///   past.  For backwards compatibility may be passed as a stringified integer.
+/// - `expiry` (required, unless ttl given) the message's expiry time as a unix epoch milliseconds
+///   timestamp.  (Unlike ttl, this cannot be passed as a stringified integer).
+/// - `data` (required) the message data, encoded in base64 (for json requests).  Max data size is
+///   76800 bytes (== 102400 in b64 encoding).  For OMQ RPC requests the value is bytes.
+/// - `namespace` (optional) a non-zero integer namespace (from -32768 to 32767) in which to store
+///   this message.  (Not accepted before the Oxen 10.x hard fork).  Messages in different
+///   namespaces are treated as separate storage boxes from untagged messages.  Different IDs have
+///   different storage properties:
+///   - namespaces divisible by 10 (e.g. 0, 60, -30) allow unauthenticated submission: that is,
+///     anyone may deposit messages into them without authentication.  Authentication is required
+///     for retrieval (and all other operations).
+///   - namespaces -30 through 30 are reserved for current and future Session message storage.
+///   - non-divisible-by-10 namespaces require authentication for all operations, including storage.
+///   Omitting the namespace is equivalent to specifying the 0 namespace.
+///
+/// Authentication parameters: these are required when storing to a namespace not divisible by 10,
+/// and must match the pubkey of the storage address.  If provided then the request will be denied
+/// if the signature does not match.  Should not be provided when depositing a message in a public
+/// receiving (i.e. divisible by 10) namespace.
+///
+/// - signature -- Ed25519 signature of ("store" || namespace || timestamp), where namespace and
+///   timestamp are the base10 expression of the namespace and timestamp values.  Must be base64
+///   encoded for json requests; binary for OMQ requests.  For non-05 type pubkeys (i.e. non session
+///   ids) the signature will be verified using `pubkey`.  For 05 pubkeys, see the following option.
+/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey` will
+///   be interpreted as an `x25519` pubkey derived from *this* given ed25519 pubkey (which must be
+///   64 hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also
+///   convert to the given `pubkey` value (without the `05` prefix) for the signature to be
+///   accepted.
+/// - sig_timestamp -- the timestamp at which this request was initiated, in milliseconds since unix
+///   epoch.  Must be within ±60s of the current time.  (For clients it is recommended to retrieve a
+///   timestamp via `info` first, to avoid client time sync issues).  If omitted, `timestamp` is
+///   used instead; it is recommended to include this value separately, particularly if a delay
+///   between message construction and message submission is possible.
 ///
 /// Returns dict of:
 /// - "swarms" dict mapping ed25519 pubkeys (in hex) of swarm members to dict values of:
 ///     - "failed" and other failure keys -- see `recursive`.
-///     - "hash": the hash of the stored message; will be an unpadded base64-encode blake2b hash
-///     of
-///       (TIMESTAMP || EXPIRY || PUBKEY || DATA), where PUBKEY is in bytes (not hex!); and DATA
-///       is in bytes (not base64).
-///     - "signature": signature of the returned "hash" value (i.e. not in decoded bytes).
-///     Returns
-///       in base64 for JSON requests, raw bytes for OMQ requests.
-///     - "already": will be true if a message with this hash was already stored (note that the
-///     hash
+///     - "hash": the hash of the stored message; will be an unpadded base64-encode blake2b hash of
+///       (TIMESTAMP || EXPIRY || PUBKEY || NAMESPACE || DATA), where PUBKEY is in bytes (not hex!);
+///       DATA is in bytes (not base64); and NAMESPACE is empty for namespace 0, and otherwise is
+///       the decimal representation of the namespace index.
+///     - "signature": signature of the returned "hash" value (i.e. not in decoded bytes).  Returned
+///       encoded in base64 for JSON requests, raw bytes for OMQ requests.
+///     - "already": will be true if a message with this hash was already stored (note that the hash
 ///       is still included and signed even if this occurs).
 ///
 struct store final : recursive {
@@ -107,10 +136,15 @@ struct store final : recursive {
     inline static constexpr size_t MAX_MESSAGE_BODY = 76'800;
 
     user_pubkey_t pubkey;
+    namespace_id msg_namespace = namespace_id::Default;
     std::chrono::system_clock::time_point timestamp;
     std::chrono::system_clock::time_point expiry;  // computed from timestamp+ttl if ttl was given
     std::string data;                              // always stored here in bytes
 
+    std::optional<std::array<unsigned char, 32>> pubkey_ed25519;
+    std::optional<std::array<unsigned char, 64>> signature;
+    std::optional<std::chrono::system_clock::time_point> sig_ts;
+
     void load_from(nlohmann::json params) override;
     void load_from(oxenc::bt_dict_consumer params) override;
     oxenc::bt_value to_bt() const override;
@@ -118,33 +152,38 @@ struct store final : recursive {
 
 /// Retrieves data from this service node. Takes keys of:
 /// - `pubkey` (required) the hex-encoded pubkey who is retrieving messages. For backwards
-/// compatibility, this can also be specified as `pubKey`
-/// - `last_hash` (optional) retrieve messages stored by this storage server since `last_hash`
-/// was stored.  Can also be specified as `lastHash`.  An empty string (or null) is treated as
-/// an omitted value.
+///   compatibility, this can also be specified as `pubKey`
+/// - `namespace` (optional) the integral message namespace from which to retrieve messages.  Each
+///   namespace forms an independent message storage for the same address.  When specified,
+///   authentication *must* be provided.  Omitting the namespace is equivalent to specifying a
+///   namespace of 0.  (Note, however, that an explicit namespace of 0 requires authentication,
+///   while an implicit namespace of 0 does not during the transition period).
+/// - `last_hash` (optional) retrieve messages stored by this storage server since `last_hash` was
+///   stored.  Can also be specified as `lastHash`.  An empty string (or null) is treated as an
+///   omitted value.
 ///
-/// Authentication parameters: these are currently optional during a transition period, and will
-/// eventually become required.  New clients should always pass them.  *If* provided then the
-/// request will be denied if the signature does not match.  If omitted, during the transition
-/// period, then messages will be retrieved without authentication.
+/// Authentication parameters: these are optional during a transition period, up until Oxen
+/// hard-fork 19, and become required starting there.  During the transition period, *if* provided
+/// then the request will be denied if the signature does not match.  If omitted, during the
+/// transition period, then messages will be retrieved without authentication.
 ///
 /// - timestamp -- the timestamp at which this request was initiated, in milliseconds since unix
-/// epoch.  Must be within ±60s of the current time.  (For clients it is recommended to retrieve
-/// a timestamp via `info` first, to avoid client time sync issues).
-/// - signature -- Ed25519 signature of ("retrieve" || timestamp), where timestamp is the base10
-/// expression of the timestamp value.  Muust be base64 encoded for json requests; binary for
-/// OMQ requests.
-/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey`
-/// will be interpreted as an `x25519` pubkey derived from this given ed25519 pubkey (which must
-/// be 64 hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also
-/// convert to the given `pubkey` value (without the `05` prefix).
+/// - signature -- Ed25519 signature of ("retrieve" || namespace || timestamp) (if using a non-0
+///   namespace), or ("retrieve" || timestamp) when fetching from the default namespace.  Both
+///   namespace and timestamp are the base10 expressions of the relevant values.  Must be base64
+///   encoded for json requests; binary for OMQ requests.
+/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey` will
+///   be interpreted as an `x25519` pubkey derived from this given ed25519 pubkey (which must be 64
+///   hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also convert
+///   to the given `pubkey` value (without the `05` prefix).
 struct retrieve final : endpoint {
     static constexpr auto names() { return NAMES("retrieve"); }
 
     user_pubkey_t pubkey;
+    namespace_id msg_namespace{0};
     std::optional<std::string> last_hash;
 
-    bool check_signature = false;
+    bool check_signature = false;  // For transition; delete this once we require sigs always
     std::optional<std::array<unsigned char, 32>> pubkey_ed25519;
     std::chrono::system_clock::time_point timestamp;
     std::array<unsigned char, 64> signature;
@@ -157,8 +196,8 @@ struct retrieve final : endpoint {
 ///
 /// Returns:
 /// - `version` the version of this storage server as a 3-element array, e.g. [2,1,1]
-/// - `timestamp` the current time (in milliseconds since unix epoch); clients are recommended
-/// to use this rather than local time, especially when submitting delete requests.
+/// - `timestamp` the current time (in milliseconds since unix epoch); clients are recommended to
+///   use this rather than local time, especially when submitting delete requests.
 ///
 struct info final : no_args {
     static constexpr auto names() { return NAMES("info"); }
@@ -169,25 +208,25 @@ struct info final : no_args {
 ///
 /// Takes parameters of:
 /// - pubkey -- the pubkey whose messages shall be deleted, in hex (66) or bytes (33)
-/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey`
-/// will be interpreted as an `x25519` pubkey derived from this given ed25519 pubkey (which must
-/// be 64 hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also
-/// convert to the given `pubkey` value (without the `05` prefix).
-/// - messages -- array of message hash strings (as provided by the storage server) to delete
-/// - signature -- Ed25519 signature of ("delete" || messages...); this signs the value
-/// constructed by concatenating "delete" and all `messages` values, using `pubkey` to sign.
-/// Must be base64 encoded for json requests; binary for OMQ requests.
+/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey` will
+///   be interpreted as an `x25519` pubkey derived from this given ed25519 pubkey (which must be 64
+///   hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also convert
+///   to the given `pubkey` value (without the `05` prefix).
+/// - messages -- array of message hash strings (as provided by the storage server) to delete.
+///   Message IDs can be from any message namespace(s).
+/// - signature -- Ed25519 signature of ("delete" || messages...); this signs the value constructed
+///   by concatenating "delete" and all `messages` values, using `pubkey` to sign.  Must be base64
+///   encoded for json requests; binary for OMQ requests.
 ///
 /// Returns dict of:
 /// - "swarms" dict mapping ed25519 pubkeys (in hex) of swarm members to dict values of:
 ///     - "failed" and other failure keys -- see `recursive`.
-///     - "deleted": list of hashes of messages that were found and deleted, sorted by ascii
-///     value
+///     - "deleted": list of hashes of messages that were found and deleted, sorted by ascii value
 ///     - "signature": signature of:
 ///             ( PUBKEY_HEX || RMSG[0] || ... || RMSG[N] || DMSG[0] || ... || DMSG[M] )
-///       where RMSG are the requested deletion hashes and DMSG are the actual deletion hashes
-///       (note that DMSG... and RMSG... will not necessarily be in the same order or of the
-///       same length). The signature uses the node's ed25519 pubkey.
+///       where RMSG are the requested deletion hashes and DMSG are the actual deletion hashes (note
+///       that DMSG... and RMSG... will not necessarily be in the same order or of the same length).
+///       The signature uses the node's ed25519 pubkey.
 struct delete_msgs final : recursive {
     static constexpr auto names() { return NAMES("delete"); }
 
@@ -201,34 +240,75 @@ struct delete_msgs final : recursive {
     oxenc::bt_value to_bt() const override;
 };
 
+struct namespace_all_t {};
+inline constexpr namespace_all_t namespace_all{};
+// Variant for holding unspecified, integer, or "all" namespace input.  Unspecified is generally the
+// same as an integer of 0 in effect, but the distinction *does* matter for the signature (which
+// matches the given arguments, not the implied value).
+using namespace_var = std::variant<std::monostate, namespace_id, namespace_all_t>;
+
+constexpr bool is_all(const namespace_var& ns) {
+    return std::holds_alternative<namespace_all_t>(ns);
+}
+constexpr bool is_omitted(const namespace_var& ns) {
+    return std::holds_alternative<std::monostate>(ns);
+}
+
+// Returns the implied namespace from a namespace_var containing either a monostate (implied
+// namespace 0) or specific namespace.  Should not be called on a variant containing an "all" value.
+constexpr namespace_id ns_or_default(const namespace_var& ns) {
+    if (auto* id = std::get_if<namespace_id>(&ns))
+        return *id;
+    return namespace_id::Default;
+}
+
+// Returns the representation of a provided namespace variant that should have been used in a
+// request signature, which is:
+// - empty string if namespace unspecified
+// - "all" if given as all namespaces
+// - "NN" for some explicitly given numeric namespace NN
+inline std::string signature_value(const namespace_var& ns) {
+    return is_omitted(ns) ? ""s : is_all(ns) ? "all"s : to_string(var::get<namespace_id>(ns));
+}
+
 /// Deletes all messages owned by the given pubkey on this SN and broadcasts the delete request
 /// to all other swarm members.
 ///
 /// Takes parameters of:
 /// - pubkey -- the pubkey whose messages shall be deleted, in hex (66) or bytes (33)
-/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey`
-/// will be interpreted as an `x25519` pubkey derived from this given ed25519 pubkey (which must
-/// be 64 hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also
-/// convert to the given `pubkey` value (without the `05` prefix).
+/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey` will
+///   be interpreted as an `x25519` pubkey derived from this given ed25519 pubkey (which must be 64
+///   hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also convert
+///   to the given `pubkey` value (without the `05` prefix).
+/// - namespace -- (optional) the message namespace from which to delete messages.  This is either
+///   an integer to delete messages from a specific namespace, or the string "all" to delete all
+///   messages from all namespaces.  If omitted, messages are deleted from the default namespace
+///   only (namespace 0).
 /// - timestamp -- the timestamp at which this request was initiated, in milliseconds since unix
-///   epoch.  Must be within ±60s of the current time.  (For clients it is recommended to
-///   retrieve a timestamp via `info` first, to avoid client time sync issues).
-/// - signature -- an Ed25519 signature of ( "delete_all" || timestamp ), signed by the ed25519
-/// pubkey in `pubkey` (omitting the leading prefix).  Must be base64 encoded for json requests;
-/// binary for OMQ requests.
+///   epoch.  Must be within ±60s of the current time.  (For clients it is recommended to retrieve a
+///   timestamp via `info` first, to avoid client time sync issues).
+/// - signature -- an Ed25519 signature of ( "delete_all" || namespace || timestamp ), where
+///   `namespace` is the stringified version of the given namespace parameter (i.e. "0" or "-42" or
+///   "all"), or the empty string if namespace was not given.  The signature must be signed by the
+///   ed25519 pubkey in `pubkey` (omitting the leading prefix).  Must be base64 encoded for json
+///   requests; binary for OMQ requests.
 ///
 /// Returns dict of:
 /// - "swarms" dict mapping ed25519 pubkeys (in hex) of swarm members to dict values of:
 ///     - "failed" and other failure keys -- see `recursive`.
-///     - "deleted": hashes of deleted messages, sorted by ascii value
-///     - "signature": signature of ( PUBKEY_HEX || TIMESTAMP || DELETEDHASH[0] || ... ||
-///     DELETEDHASH[N] ), signed
-///       by the node's ed25519 pubkey.
+///     - "deleted": if deleting from a single namespace this is a list of hashes of deleted
+///       messages from the namespace, sorted by ascii value.  If deleting from all namespaces this
+///       is a dict of `{ namespace => [sorted list of hashes] }` key-value pairs.
+///     - "signature": signature of:
+///           ( PUBKEY_HEX || TIMESTAMP || DELETEDHASH[0] || ... || DELETEDHASH[N] )
+///       signed by the node's ed25519 pubkey.  When doing a multi-namespace delete the DELETEDHASH
+///       values are totally ordered (i.e. among all the hashes deleted regardless of namespace)
 struct delete_all final : recursive {
     static constexpr auto names() { return NAMES("delete_all"); }
 
     user_pubkey_t pubkey;
     std::optional<std::array<unsigned char, 32>> pubkey_ed25519;
+    namespace_var msg_namespace;
     std::chrono::system_clock::time_point timestamp;
     std::array<unsigned char, 64> signature;
 
@@ -242,29 +322,38 @@ struct delete_all final : recursive {
 ///
 /// Takes parameters of:
 /// - pubkey -- the pubkey whose messages shall be deleted, in hex (66) or bytes (33)
-/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey`
-/// will be interpreted as an `x25519` pubkey derived from this given ed25519 pubkey (which must
-/// be 64 hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also
-/// convert to the given `pubkey` value (without the `05` prefix).
-/// - before -- the timestamp (in milliseconds since unix epoch) for deletion; all stored
-/// messages
+/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey` will
+///   be interpreted as an `x25519` pubkey derived from this given ed25519 pubkey (which must be 64
+///   hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also convert
+///   to the given `pubkey` value (without the `05` prefix).
+/// - namespace -- (optional) the message namespace from which to delete messages.  This is either
+///   an integer to delete messages from a specific namespace, or the string "all" to delete
+///   messages from all namespaces.  If omitted, messages are deleted from the default namespace
+///   only (namespace 0).
+/// - before -- the timestamp (in milliseconds since unix epoch) for deletion; all stored messages
 ///   with timestamps <= this value will be deleted.  Should be <= now, but tolerance acceptance
 ///   allows it to be <= 60s from now.
-/// - signature -- Ed25519 signature of ("delete_before" || before), signed by `pubkey`.  Must
-/// be base64 encoded (json) or bytes (OMQ).
+/// - signature -- Ed25519 signature of ("delete_before" || namespace || before), signed by
+///   `pubkey`.  Must be base64 encoded (json) or bytes (OMQ).  `namespace` is the stringified
+///   version of the given namespace parameter (i.e. "0" or "-42" or "all"), or the empty string if
+///   namespace was not given.
 ///
 /// Returns dict of:
 /// - "swarms" dict mapping ed25519 pubkeys (in hex) of swarm members to dict values of:
 ///     - "failed" and other failure keys -- see `recursive`.
-///     - "deleted": hashes of deleted messages, sorted by ascii value
-///     - "signature": signature of ( PUBKEY_HEX || BEFORE || DELETEDHASH[0] || ... ||
-///     DELETEDHASH[N] ), signed
-///       by the node's ed25519 pubkey.
+///     - "deleted": if deleting from a single namespace this is a list of hashes of deleted
+///       messages from the namespace, sorted by ascii value.  If deleting from all namespaces this
+///       is a dict of `{ namespace => [sorted list of hashes] }` key-value pairs.
+///     - "signature": signature of
+///           ( PUBKEY_HEX || BEFORE || DELETEDHASH[0] || ... || DELETEDHASH[N] )
+///       signed by the node's ed25519 pubkey.  When doing a multi-namespace delete the DELETEDHASH
+///       values are totally ordered (i.e. among all the hashes deleted regardless of namespace)
 struct delete_before final : recursive {
     static constexpr auto names() { return NAMES("delete_before"); }
 
     user_pubkey_t pubkey;
     std::optional<std::array<unsigned char, 32>> pubkey_ed25519;
+    namespace_var msg_namespace;
     std::chrono::system_clock::time_point before;
     std::array<unsigned char, 64> signature;
 
@@ -278,31 +367,41 @@ struct delete_before final : recursive {
 /// shorten the expiry of any messages that have expiries after the requested value.
 ///
 /// Takes parameters of:
-/// - pubkey -- the pubkey whose messages shall have their expiries reduced, in hex (66) or
-/// bytes (33)
-/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey`
-/// will be interpreted as an `x25519` pubkey derived from this given ed25519 pubkey (which must
-/// be 64 hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also
-/// convert to the given `pubkey` value (without the `05` prefix).
+/// - pubkey -- the pubkey whose messages shall have their expiries reduced, in hex (66) or bytes
+///   (33)
+/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey` will
+///   be interpreted as an `x25519` pubkey derived from this given ed25519 pubkey (which must be 64
+///   hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also convert
+///   to the given `pubkey` value (without the `05` prefix).
+/// - namespace -- (optional) the message namespace from which to change message expiries.  This is
+///   either an integer to expire messages from a specific namespace, or the string "all" to update
+///   messages in all namespaces.  If omitted, the update applies only to messages from the default
+///   namespace (namespace 0).
 /// - expiry -- the new expiry timestamp (milliseconds since unix epoch).  Should be >= now, but
 ///   tolerance acceptance allows >= 60s ago.
 /// - signature -- signature of ("expire_all" || expiry), signed by `pubkey`.  Must be base64
-/// encoded (json) or bytes (OMQ).
+///   encoded (json) or bytes (OMQ).
 ///
 /// Returns dict of:
 /// - "swarms" dict mapping ed25519 pubkeys (in hex) of swarm members to dict values of:
 ///     - "failed" and other failure keys -- see `recursive`.
-///     - "updated": list of (ascii-sorted) hashes that had their expiries updated to `expiry`;
-///       messages that did not exist or that already had an expiry <= the given expiry are not
-///       included.
-///     - "signature": signature of ( PUBKEY_HEX || EXPIRY || UPDATED[0] || ... || UPDATED[N] ),
-///     signed
-///       by the node's ed25519 pubkey.
+///     - "updated":
+///         - if deleting from a single namespace then this is a list of (ascii-sorted) hashes that
+///           had their expiries updated to `expiry`; messages that did not exist or that already
+///           had an expiry <= the given expiry are not included.
+///         - otherwise (i.e. namespace="all") this is a dict of `{ namespace => [sorted hashes] }`
+///           pairs of updated-expiry message hashes.
+///     - "signature": signature of
+///           ( PUBKEY_HEX || EXPIRY || UPDATED[0] || ... || UPDATED[N] )
+///       signed by the node's ed25519 pubkey.  When doing a multi-namespace expiry update the
+///       UPDATED values are totally ordered (i.e. among all the messages updated regardless of
+///       namespace)
 struct expire_all final : recursive {
     static constexpr auto names() { return NAMES("expire_all"); }
 
     user_pubkey_t pubkey;
     std::optional<std::array<unsigned char, 32>> pubkey_ed25519;
+    namespace_var msg_namespace;
     std::chrono::system_clock::time_point expiry;
     std::array<unsigned char, 64> signature;
 
@@ -315,17 +414,19 @@ struct expire_all final : recursive {
 /// request to all other swarm members.
 ///
 /// Takes parameters of:
-/// - pubkey -- the pubkey whose messages shall have their expiries reduced, in hex (66) or
-/// bytes (33)
-/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey`
-/// will be interpreted as an `x25519` pubkey derived from this given ed25519 pubkey (which must
-/// be 64 hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also
-/// convert to the given `pubkey` value (without the `05` prefix).
-/// - messages -- array of message hash strings (as provided by the storage server) to update
+/// - pubkey -- the pubkey whose messages shall have their expiries reduced, in hex (66) or bytes
+///   (33)
+/// - pubkey_ed25519 if provided *and* the pubkey has a type 05 (i.e. Session id) then `pubkey` will
+///   be interpreted as an `x25519` pubkey derived from this given ed25519 pubkey (which must be 64
+///   hex characters or 32 bytes).  *This* pubkey should be used for signing, but must also convert
+///   to the given `pubkey` value (without the `05` prefix).
+/// - messages -- array of message hash strings (as provided by the storage server) to update.
+///   Messages can be from any namespace(s).
 /// - expiry -- the new expiry timestamp (milliseconds since unix epoch).  Must be >= 60s ago.
-/// - signature -- Ed25519 signature of ("expire" || expiry || messages[0] || ... ||
-/// messages[N]) (where `expiry` is the expiry timestamp expressed as a string).  Must be base64
-/// encoded (json) or bytes (OMQ).
+/// - signature -- Ed25519 signature of:
+///       ("expire" || expiry || messages[0] || ... || messages[N])
+///   where `expiry` is the expiry timestamp expressed as a string.  The signature must be base64
+///   encoded (json) or bytes (bt).
 ///
 ///
 /// Returns dict of:
@@ -334,10 +435,9 @@ struct expire_all final : recursive {
 ///     - "updated": ascii-sorted list of hashes of messages that had their expiries updated
 ///       (messages that already had an expiry <= the given expiry are not included).
 ///     - "signature": signature of:
-///             ( PUBKEY_HEX || EXPIRY || RMSG[0] || ... || RMSG[N] || UMSG[0] || ... || UMSG[M]
-///             )
-///       where RMSG are the requested expiry hashes and UMSG are the actual updated hashes.
-///       The signature uses the node's ed25519 pubkey.
+///             ( PUBKEY_HEX || EXPIRY || RMSG[0] || ... || RMSG[N] || UMSG[0] || ... || UMSG[M] )
+///       where RMSG are the requested expiry hashes and UMSG are the actual updated hashes.  The
+///       signature uses the node's ed25519 pubkey.
 struct expire_msgs final : recursive {
     static constexpr auto names() { return NAMES("expire"); }
 
@@ -372,8 +472,8 @@ struct get_swarm final : endpoint {
 /// - `params` (optional) dict of parameters to forward to oxend.  Can be omitted or null if no
 ///   parameters should be passed.
 ///
-/// See oxend rpc documentation (or the oxen-core/src/rpc/core_rpc_server_command_defs.h file)
-/// for information on using these oxend rpc endpoints.
+/// See oxend rpc documentation (or the oxen-core/src/rpc/core_rpc_server_command_defs.h file) for
+/// information on using these oxend rpc endpoints.
 struct oxend_request final : endpoint {
     static constexpr auto names() { return NAMES("oxend_request"); }
 

httpserver/https_server.cpp

@@ -685,13 +685,13 @@ void HTTPSServer::process_storage_rpc_req(HttpRequest& req, HttpResponse& res) {
                                                     debug,
                                                     "Responding to a client request after {}",
                                                     util::friendly_duration(
-                                                            std::chrono::steady_clock::now()
-                                                            - started));
+                                                            std::chrono::steady_clock::now() -
+                                                            started));
                                             queue_response(std::move(data), std::move(response));
                                         });
                             } catch (const std::exception& e) {
-                                auto error = "Exception caught with processing client request: "s
-                                           + e.what();
+                                auto error = "Exception caught with processing client request: "s +
+                                             e.what();
                                 OXEN_LOG(critical, "{}", error);
                                 queue_response(
                                         std::move(data), {http::INTERNAL_SERVER_ERROR, error});
@@ -728,8 +728,8 @@ void HTTPSServer::process_onion_req_v2(HttpRequest& req, HttpResponse& res) {
                                                 res.status.first,
                                                 res.status.second,
                                                 util::friendly_duration(
-                                                        std::chrono::steady_clock::now()
-                                                        - started));
+                                                        std::chrono::steady_clock::now() -
+                                                        started));
                                         queue_response(std::move(data), std::move(res));
                                     },
                                     0,  // hopno

httpserver/onion_processing.cpp

@@ -127,8 +127,8 @@ ParsedInfo process_ciphertext_v2(
 }
 
 bool is_onion_url_target_allowed(std::string_view target) {
-    return (util::starts_with(target, "/loki/") || util::starts_with(target, "/oxen/"))
-        && util::ends_with(target, "/lsrpc") && target.find('?') == std::string::npos;
+    return (util::starts_with(target, "/loki/") || util::starts_with(target, "/oxen/")) &&
+           util::ends_with(target, "/lsrpc") && target.find('?') == std::string::npos;
 }
 
 /// We are expecting a payload of the following shape:
@@ -187,8 +187,8 @@ std::ostream& operator<<(std::ostream& os, const RelayToServerInfo& d) {
 }
 
 bool operator==(const RelayToServerInfo& lhs, const RelayToServerInfo& rhs) {
-    return (lhs.protocol == rhs.protocol) && (lhs.host == rhs.host) && (lhs.port == rhs.port)
-        && (lhs.target == rhs.target) && (lhs.payload == rhs.payload);
+    return (lhs.protocol == rhs.protocol) && (lhs.host == rhs.host) && (lhs.port == rhs.port) &&
+           (lhs.target == rhs.target) && (lhs.payload == rhs.payload);
 }
 
 std::ostream& operator<<(std::ostream& os, const RelayToNodeInfo& d) {
@@ -201,8 +201,8 @@ std::ostream& operator<<(std::ostream& os, const RelayToNodeInfo& d) {
 }
 
 bool operator==(const RelayToNodeInfo& a, const RelayToNodeInfo& b) {
-    return std::tie(a.ciphertext, a.ephemeral_key, a.enc_type, a.next_node)
-        == std::tie(b.ciphertext, b.ephemeral_key, b.enc_type, b.next_node);
+    return std::tie(a.ciphertext, a.ephemeral_key, a.enc_type, a.next_node) ==
+           std::tie(b.ciphertext, b.ephemeral_key, b.enc_type, b.next_node);
 }
 
 }  // namespace oxen

httpserver/oxend_rpc.cpp

@@ -42,8 +42,8 @@ oxend_seckeys get_sn_privkeys(
                                 try {
                                     if (!success || data.size() < 2) {
                                         throw std::runtime_error{
-                                                "oxend SN keys request failed: "
-                                                + (data.empty() ? "no data received" : data[0])};
+                                                "oxend SN keys request failed: " +
+                                                (data.empty() ? "no data received" : data[0])};
                                     }
                                     auto r = nlohmann::json::parse(data[1]);
 

httpserver/reachability_testing.cpp

@@ -16,8 +16,8 @@ static void check_incoming_tests_impl(
         detail::incoming_test_state& incoming) {
     const auto elapsed = now - std::max(startup, incoming.last_test);
     bool failing = elapsed > reachability_testing::MAX_TIME_WITHOUT_PING;
-    bool whine = failing != incoming.was_failing
-              || (failing && now - incoming.last_whine > reachability_testing::WHINING_INTERVAL);
+    bool whine = failing != incoming.was_failing ||
+                 (failing && now - incoming.last_whine > reachability_testing::WHINING_INTERVAL);
 
     incoming.was_failing = failing;
 
@@ -58,9 +58,8 @@ std::optional<sn_record> reachability_testing::next_random(
         const Swarm& swarm, const clock::time_point& now, bool requeue) {
     if (next_general_test > now)
         return std::nullopt;
-    next_general_test =
-            now
-            + std::chrono::duration_cast<clock::duration>(fseconds(TESTING_INTERVAL(util::rng())));
+    next_general_test = now + std::chrono::duration_cast<clock::duration>(
+                                      fseconds(TESTING_INTERVAL(util::rng())));
 
     // Pull the next element off the queue, but skip ourself, any that are no longer registered,
     // and any that are currently known to be failing (those are queued for testing separately).

httpserver/request_handler.cpp

@@ -16,7 +16,6 @@
 
 #include <cpr/cpr.h>
 #include <nlohmann/json.hpp>
-#include <openssl/sha.h>
 #include <oxenc/base32z.h>
 #include <oxenc/base64.h>
 #include <oxenc/hex.h>
@@ -53,15 +52,13 @@ namespace {
         json snodes_json = json::array();
         for (const auto& sn : swarm.snodes) {
             snodes_json.push_back(
-                    json{{"address",
-                          oxenc::to_base32z(sn.pubkey_legacy.view())
-                                  + ".snode"},  // Deprecated, use pubkey_legacy instead
+                    json{{"address",  // Deprecated, use pubkey_legacy instead
+                          oxenc::to_base32z(sn.pubkey_legacy.view()) + ".snode"},
                          {"pubkey_legacy", sn.pubkey_legacy.hex()},
                          {"pubkey_x25519", sn.pubkey_x25519.hex()},
                          {"pubkey_ed25519", sn.pubkey_ed25519.hex()},
-                         {"port",
-                          std::to_string(sn.port)},  // Deprecated port (as a string) for backwards
-                                                     // compat; use "port_https" instead
+                         {"port",  // Deprecated string port for backwards compat; prefer https_port
+                          std::to_string(sn.port)},
                          {"port_https", sn.port},
                          {"port_omq", sn.omq_port},
                          {"ip", sn.ip}});
@@ -78,8 +75,8 @@ namespace {
         const auto& pk_raw = pk.raw();
         if (pk_raw.empty())
             return "(none)";
-        return oxenc::to_hex(pk_raw.begin(), pk_raw.begin() + 2) + u8"…"
-             + oxenc::to_hex(std::prev(pk_raw.end()), pk_raw.end());
+        return oxenc::to_hex(pk_raw.begin(), pk_raw.begin() + 2) + u8"…" +
+               oxenc::to_hex(std::prev(pk_raw.end()), pk_raw.end());
     }
 
     template <typename RPC>
@@ -130,10 +127,8 @@ namespace {
             s += std::string_view{val}.size();
         else {
             static_assert(
-                    std::is_same_v<
-                            T,
-                            std::vector<
-                                    std::string>> || std::is_same_v<T, std::vector<std::string_view>>);
+                    std::is_same_v<T, std::vector<std::string>> ||
+                    std::is_same_v<T, std::vector<std::string_view>>);
             for (auto& v : val)
                 s += v.size();
         }
@@ -155,10 +150,8 @@ namespace {
             result += std::string_view{val};
         else {
             static_assert(
-                    std::is_same_v<
-                            T,
-                            std::vector<
-                                    std::string>> || std::is_same_v<T, std::vector<std::string_view>>);
+                    std::is_same_v<T, std::vector<std::string>> ||
+                    std::is_same_v<T, std::vector<std::string_view>>);
             for (auto& v : val)
                 result += v;
         }
@@ -203,20 +196,19 @@ namespace {
 
             // Verify that the given ed pubkey actually converts to the x25519 pubkey
             std::array<unsigned char, crypto_scalarmult_curve25519_BYTES> xpk;
-            if (crypto_sign_ed25519_pk_to_curve25519(xpk.data(), pk) != 0
-                || std::memcmp(xpk.data(), raw.data(), crypto_scalarmult_curve25519_BYTES) != 0) {
+            if (crypto_sign_ed25519_pk_to_curve25519(xpk.data(), pk) != 0 ||
+                std::memcmp(xpk.data(), raw.data(), crypto_scalarmult_curve25519_BYTES) != 0) {
                 OXEN_LOG(debug, "Signature verification failed: ed -> x conversion did not match");
                 return false;
             }
         } else
             pk = reinterpret_cast<const unsigned char*>(raw.data());
 
-        bool verified = 0
-                     == crypto_sign_verify_detached(
-                                sig.data(),
-                                reinterpret_cast<const unsigned char*>(data.data()),
-                                data.size(),
-                                pk);
+        bool verified = 0 == crypto_sign_verify_detached(
+                                     sig.data(),
+                                     reinterpret_cast<const unsigned char*>(data.data()),
+                                     data.size(),
+                                     pk);
         if (!verified)
             OXEN_LOG(debug, "Signature verification failed");
         return verified;
@@ -261,14 +253,20 @@ std::string computeMessageHash(
         system_clock::time_point timestamp,
         system_clock::time_point expiry,
         const user_pubkey_t& pubkey,
+        namespace_id ns,
         std::string_view data) {
     char netid = static_cast<char>(pubkey.type());
+    std::array<char, 20> ns_buf;
+    char* ns_buf_ptr = ns_buf.data();
+    std::string_view ns_for_hash =
+            ns != namespace_id::Default ? detail::to_hashable(to_int(ns), ns_buf_ptr) : ""sv;
     return compute_hash(
             compute_hash_blake2b_b64,
             timestamp,
             expiry,
             std::string_view{&netid, 1},
             pubkey.raw(),
+            ns_for_hash,
             data);
 }
 
@@ -413,6 +411,35 @@ void RequestHandler::process_client_req(rpc::store&& req, std::function<void(Res
         return cb(Response{http::NOT_ACCEPTABLE, "Timestamp error: check your clock"sv});
     }
 
+    if (!is_public_namespace(req.msg_namespace)) {
+        if (!req.signature) {
+            auto err = fmt::format(
+                    "store: signature required to store to namespace {}",
+                    to_int(req.msg_namespace));
+            OXEN_LOG(warn, err);
+            return cb(Response{http::UNAUTHORIZED, err});
+        }
+        if (req.timestamp < now - SIGNATURE_TOLERANCE ||
+            req.timestamp > now + SIGNATURE_TOLERANCE) {
+            OXEN_LOG(
+                    debug,
+                    "store: invalid timestamp ({}s from now)",
+                    duration_cast<seconds>(req.timestamp - now).count());
+            return cb(
+                    Response{http::NOT_ACCEPTABLE, "store timestamp too far from current time"sv});
+        }
+        if (!verify_signature(
+                    req.pubkey,
+                    req.pubkey_ed25519,
+                    *req.signature,
+                    "store",
+                    req.msg_namespace == namespace_id::Default ? "" : to_string(req.msg_namespace),
+                    req.timestamp)) {
+            OXEN_LOG(debug, "store: signature verification failed");
+            return cb(Response{http::UNAUTHORIZED, "store signature verification failed"sv});
+        }
+    }
+
     bool entry_router = req.recurse == true;
 
     auto [res, lock] = setup_recursive_request(service_node_, req, std::move(cb));
@@ -420,13 +447,19 @@ void RequestHandler::process_client_req(rpc::store&& req, std::function<void(Res
                        ? res->result["swarm"][service_node_.own_address().pubkey_ed25519.hex()]
                        : res->result;
 
-    std::string message_hash = computeMessageHash(req.timestamp, req.expiry, req.pubkey, req.data);
+    std::string message_hash =
+            computeMessageHash(req.timestamp, req.expiry, req.pubkey, req.msg_namespace, req.data);
 
     bool new_msg;
     bool success = false;
     try {
         success = service_node_.process_store(
-                message{req.pubkey, message_hash, req.timestamp, req.expiry, std::move(req.data)},
+                message{req.pubkey,
+                        message_hash,
+                        req.msg_namespace,
+                        req.timestamp,
+                        req.expiry,
+                        std::move(req.data)},
                 &new_msg);
     } catch (const std::exception& e) {
         OXEN_LOG(
@@ -463,8 +496,11 @@ void RequestHandler::process_client_req(rpc::store&& req, std::function<void(Res
 
     OXEN_LOG(
             trace,
-            "Successfully stored message {} for {}",
+            "Successfully stored message {}{} for {}",
             message_hash,
+            req.msg_namespace != namespace_id::Default
+                    ? fmt::format("[{}]", to_int(req.msg_namespace))
+                    : "",
             obfuscate_pubkey(req.pubkey));
 
     if (--res->pending == 0)
@@ -532,9 +568,20 @@ void RequestHandler::process_client_req(
         return cb(handle_wrong_swarm(req.pubkey));
 
     auto now = system_clock::now();
+
+    // At HF19 start requiring authentication for all retrievals (except legacy closed groups, which
+    // can't be authenticated for technical reasons).
+    if (service_node_.hf_at_least(HARDFORK_RETRIEVE_AUTH) &&
+        req.msg_namespace != namespace_id::LegacyClosed) {
+        if (!req.check_signature) {
+            OXEN_LOG(debug, "retrieve: request signature required as of HF19");
+            return cb(Response{http::UNAUTHORIZED, "retrieve: request signature required"sv});
+        }
+    }
+
     if (req.check_signature) {
-        if (req.timestamp < now - SIGNATURE_TOLERANCE
-            || req.timestamp > now + SIGNATURE_TOLERANCE) {
+        if (req.timestamp < now - SIGNATURE_TOLERANCE ||
+            req.timestamp > now + SIGNATURE_TOLERANCE) {
             OXEN_LOG(
                     debug,
                     "retrieve: invalid timestamp ({}s from now)",
@@ -543,7 +590,14 @@ void RequestHandler::process_client_req(
                     http::NOT_ACCEPTABLE, "retrieve timestamp too far from current time"sv});
         }
         if (!verify_signature(
-                    req.pubkey, req.pubkey_ed25519, req.signature, "retrieve", req.timestamp)) {
+                    req.pubkey,
+                    req.pubkey_ed25519,
+                    req.signature,
+                    "retrieve",
+                    req.msg_namespace != namespace_id::Default
+                            ? std::to_string(to_int(req.msg_namespace))
+                            : ""s,
+                    req.timestamp)) {
             OXEN_LOG(debug, "retrieve: signature verification failed");
             return cb(Response{http::UNAUTHORIZED, "retrieve signature verification failed"sv});
         }
@@ -551,7 +605,9 @@ void RequestHandler::process_client_req(
 
     std::vector<message> msgs;
     try {
-        msgs = service_node_.retrieve(req.pubkey, req.last_hash.value_or(""));
+        msgs = service_node_.get_db().retrieve(
+                req.pubkey, req.msg_namespace, req.last_hash.value_or(""));
+        service_node_.record_retrieve_request();
     } catch (const std::exception& e) {
         auto msg = fmt::format(
                 "Internal Server Error. Could not retrieve messages for {}",
@@ -587,6 +643,48 @@ void RequestHandler::process_client_req(rpc::info&&, std::function<void(oxen::Re
                  {"timestamp", to_epoch_ms(system_clock::now())}}});
 }
 
+namespace {
+    template <typename... SigArgs>
+    void handle_action_all_ns(
+            nlohmann::json& mine,
+            const std::string& mine_key,
+            std::vector<std::pair<namespace_id, std::string>>&& affected,
+            bool b64,
+            SigArgs&&... signature_args) {
+
+        std::sort(affected.begin(), affected.end(), [](const auto& a, const auto& b) {
+            return a.second < b.second;
+        });
+        std::vector<std::string_view> sorted_hashes;
+        sorted_hashes.reserve(affected.size());
+        for (const auto& [ns, hash] : affected)
+            sorted_hashes.emplace_back(hash);
+
+        auto sig = create_signature(std::forward<SigArgs>(signature_args)..., sorted_hashes);
+        mine["signature"] = b64 ? oxenc::to_base64(sig.begin(), sig.end()) : util::view_guts(sig);
+
+        // We've totally sorted by hash (for the signature, above), so this loop below will be
+        // appending to the sublists in sorted order:
+        auto& result = (mine[mine_key] = json::object());
+        for (auto& [ns, hash] : affected)
+            result[to_string(ns)].push_back(std::move(hash));
+    }
+
+    template <typename... SigArgs>
+    void handle_action_one_ns(
+            nlohmann::json& mine,
+            const std::string& mine_key,
+            std::vector<std::string>&& affected,
+            bool b64,
+            SigArgs&&... signature_args) {
+
+        std::sort(affected.begin(), affected.end());
+        auto sig = create_signature(std::forward<SigArgs>(signature_args)..., affected);
+        mine["signature"] = b64 ? oxenc::to_base64(sig.begin(), sig.end()) : util::view_guts(sig);
+        mine[mine_key] = std::move(affected);
+    }
+}  // namespace
+
 void RequestHandler::process_client_req(
         rpc::delete_all&& req, std::function<void(oxen::Response)> cb) {
     OXEN_LOG(debug, "processing delete_all {} request", req.recurse ? "direct" : "forwarded");
@@ -606,7 +704,12 @@ void RequestHandler::process_client_req(
     }
 
     if (!verify_signature(
-                req.pubkey, req.pubkey_ed25519, req.signature, "delete_all", req.timestamp)) {
+                req.pubkey,
+                req.pubkey_ed25519,
+                req.signature,
+                "delete_all",
+                signature_value(req.msg_namespace),
+                req.timestamp)) {
         OXEN_LOG(debug, "delete_all: signature verification failed");
         return cb(Response{http::UNAUTHORIZED, "delete_all signature verification failed"sv});
     }
@@ -619,17 +722,27 @@ void RequestHandler::process_client_req(
                        ? res->result["swarm"][service_node_.own_address().pubkey_ed25519.hex()]
                        : res->result;
 
-    if (auto deleted = service_node_.delete_all_messages(req.pubkey)) {
-        std::sort(deleted->begin(), deleted->end());
-        auto sig =
-                create_signature(ed25519_sk_, req.pubkey.prefixed_hex(), req.timestamp, *deleted);
-        mine["deleted"] = std::move(*deleted);
-        mine["signature"] =
-                req.b64 ? oxenc::to_base64(sig.begin(), sig.end()) : util::view_guts(sig);
+    if (is_all(req.msg_namespace)) {
+        handle_action_all_ns(
+                mine,
+                "deleted",
+                service_node_.get_db().delete_all(req.pubkey),
+                req.b64,
+                ed25519_sk_,
+                req.pubkey.prefixed_hex(),
+                req.timestamp);
+
     } else {
-        mine["failed"] = true;
-        mine["query_failure"] = true;
+        handle_action_one_ns(
+                mine,
+                "deleted",
+                service_node_.get_db().delete_all(req.pubkey, ns_or_default(req.msg_namespace)),
+                req.b64,
+                ed25519_sk_,
+                req.pubkey.prefixed_hex(),
+                req.timestamp);
     }
+
     if (req.recurse)
         mine["t"] = to_epoch_ms(now);
 
@@ -656,16 +769,11 @@ void RequestHandler::process_client_req(rpc::delete_msgs&& req, std::function<vo
                        ? res->result["swarm"][service_node_.own_address().pubkey_ed25519.hex()]
                        : res->result;
 
-    if (auto deleted = service_node_.delete_messages(req.pubkey, req.messages)) {
-        std::sort(deleted->begin(), deleted->end());
-        auto sig = create_signature(ed25519_sk_, req.pubkey.prefixed_hex(), req.messages, *deleted);
-        mine["deleted"] = std::move(*deleted);
-        mine["signature"] =
-                req.b64 ? oxenc::to_base64(sig.begin(), sig.end()) : util::view_guts(sig);
-    } else {
-        mine["failed"] = true;
-        mine["query_failure"] = true;
-    }
+    auto deleted = service_node_.get_db().delete_by_hash(req.pubkey, req.messages);
+    std::sort(deleted.begin(), deleted.end());
+    auto sig = create_signature(ed25519_sk_, req.pubkey.prefixed_hex(), req.messages, deleted);
+    mine["deleted"] = std::move(deleted);
+    mine["signature"] = req.b64 ? oxenc::to_base64(sig.begin(), sig.end()) : util::view_guts(sig);
     if (req.recurse)
         mine["t"] = to_epoch_ms(std::chrono::system_clock::now());
 
@@ -690,7 +798,12 @@ void RequestHandler::process_client_req(
     }
 
     if (!verify_signature(
-                req.pubkey, req.pubkey_ed25519, req.signature, "delete_before", req.before)) {
+                req.pubkey,
+                req.pubkey_ed25519,
+                req.signature,
+                "delete_before",
+                signature_value(req.msg_namespace),
+                req.before)) {
         OXEN_LOG(debug, "delete_before: signature verification failed");
         return cb(Response{http::UNAUTHORIZED, "delete_before signature verification failed"sv});
     }
@@ -703,15 +816,26 @@ void RequestHandler::process_client_req(
                        ? res->result["swarm"][service_node_.own_address().pubkey_ed25519.hex()]
                        : res->result;
 
-    if (auto deleted = service_node_.delete_messages_before(req.pubkey, req.before)) {
-        std::sort(deleted->begin(), deleted->end());
-        auto sig = create_signature(ed25519_sk_, req.pubkey.prefixed_hex(), req.before, *deleted);
-        mine["deleted"] = std::move(*deleted);
-        mine["signature"] =
-                req.b64 ? oxenc::to_base64(sig.begin(), sig.end()) : util::view_guts(sig);
+    if (is_all(req.msg_namespace)) {
+        handle_action_all_ns(
+                mine,
+                "deleted",
+                service_node_.get_db().delete_by_timestamp(req.pubkey, req.before),
+                req.b64,
+                ed25519_sk_,
+                req.pubkey.prefixed_hex(),
+                req.before);
+
     } else {
-        mine["failed"] = true;
-        mine["query_failure"] = true;
+        handle_action_one_ns(
+                mine,
+                "deleted",
+                service_node_.get_db().delete_by_timestamp(
+                        req.pubkey, ns_or_default(req.msg_namespace), req.before),
+                req.b64,
+                ed25519_sk_,
+                req.pubkey.prefixed_hex(),
+                req.before);
     }
     if (req.recurse)
         mine["t"] = to_epoch_ms(now);
@@ -750,15 +874,25 @@ void RequestHandler::process_client_req(rpc::expire_all&& req, std::function<voi
                        ? res->result["swarm"][service_node_.own_address().pubkey_ed25519.hex()]
                        : res->result;
 
-    if (auto updated = service_node_.update_all_expiries(req.pubkey, req.expiry)) {
-        std::sort(updated->begin(), updated->end());
-        auto sig = create_signature(ed25519_sk_, req.pubkey.prefixed_hex(), req.expiry, *updated);
-        mine["updated"] = std::move(*updated);
-        mine["signature"] =
-                req.b64 ? oxenc::to_base64(sig.begin(), sig.end()) : util::view_guts(sig);
+    if (is_all(req.msg_namespace)) {
+        handle_action_all_ns(
+                mine,
+                "updated",
+                service_node_.get_db().update_all_expiries(req.pubkey, req.expiry),
+                req.b64,
+                ed25519_sk_,
+                req.pubkey.prefixed_hex(),
+                req.expiry);
     } else {
-        mine["failed"] = true;
-        mine["query_failure"] = true;
+        handle_action_one_ns(
+                mine,
+                "updated",
+                service_node_.get_db().update_all_expiries(
+                        req.pubkey, ns_or_default(req.msg_namespace), req.expiry),
+                req.b64,
+                ed25519_sk_,
+                req.pubkey.prefixed_hex(),
+                req.expiry);
     }
     if (req.recurse)
         mine["t"] = to_epoch_ms(now);
@@ -800,17 +934,12 @@ void RequestHandler::process_client_req(rpc::expire_msgs&& req, std::function<vo
                        ? res->result["swarm"][service_node_.own_address().pubkey_ed25519.hex()]
                        : res->result;
 
-    if (auto updated = service_node_.update_messages_expiry(req.pubkey, req.messages, req.expiry)) {
-        std::sort(updated->begin(), updated->end());
-        auto sig = create_signature(
-                ed25519_sk_, req.pubkey.prefixed_hex(), req.expiry, req.messages, *updated);
-        mine["updated"] = std::move(*updated);
-        mine["signature"] =
-                req.b64 ? oxenc::to_base64(sig.begin(), sig.end()) : util::view_guts(sig);
-    } else {
-        mine["failed"] = true;
-        mine["query_failure"] = true;
-    }
+    auto updated = service_node_.get_db().update_expiry(req.pubkey, req.messages, req.expiry);
+    std::sort(updated.begin(), updated.end());
+    auto sig = create_signature(
+            ed25519_sk_, req.pubkey.prefixed_hex(), req.expiry, req.messages, updated);
+    mine["updated"] = std::move(updated);
+    mine["signature"] = req.b64 ? oxenc::to_base64(sig.begin(), sig.end()) : util::view_guts(sig);
     if (req.recurse)
         mine["t"] = to_epoch_ms(now);
 
@@ -875,7 +1004,7 @@ void RequestHandler::process_client_req(
 Response RequestHandler::process_retrieve_all() {
     std::vector<message> msgs;
     try {
-        msgs = service_node_.get_all_messages();
+        msgs = service_node_.get_db().retrieve_all();
     } catch (const std::exception& e) {
         return {http::INTERNAL_SERVER_ERROR, "could not retrieve all messages"s};
     }
@@ -924,8 +1053,8 @@ void RequestHandler::process_storage_test_req(
 
                     auto [status, answer] =
                             service_node_.process_storage_test_req(height, tester, hash);
-                    if (status == MessageTestStatus::RETRY && elapsed < TEST_RETRY_PERIOD
-                        && !service_node_.shutting_down())
+                    if (status == MessageTestStatus::RETRY && elapsed < TEST_RETRY_PERIOD &&
+                        !service_node_.shutting_down())
                         return;  // Still retrying so wait for the next call
                     service_node_.omq_server()->cancel_timer(*timer);
                     callback(status, std::move(answer), elapsed);
@@ -1046,8 +1175,8 @@ void RequestHandler::process_onion_req(RelayToServerInfo&& info, OnionRequestMet
     OXEN_LOG(debug, "We are to forward the request to url: {}{}", info.host, info.target);
 
     // Forward the request to url but only if it ends in `/lsrpc`
-    if (!(info.protocol == "http" || info.protocol == "https")
-        || !is_onion_url_target_allowed(info.target))
+    if (!(info.protocol == "http" || info.protocol == "https") ||
+        !is_onion_url_target_allowed(info.target))
         return data.cb(wrap_proxy_response(
                 {http::BAD_REQUEST, "Invalid url"s}, data.ephem_key, data.enc_type));
 

httpserver/request_handler.h

@@ -104,10 +104,10 @@ std::string compute_hash(Func hasher, const T&... args) {
     // value can be when stringified).
     std::array<
             char,
-            (0 + ...
-             + (std::is_integral_v<T> || std::is_same_v<T, std::chrono::system_clock::time_point>
-                        ? 20
-                        : 0))>
+            (0 + ... +
+             (std::is_integral_v<T> || std::is_same_v<T, std::chrono::system_clock::time_point>
+                      ? 20
+                      : 0))>
             buffer;
     auto* b = buffer.data();
     return hasher({detail::to_hashable(args, b)...});
@@ -118,6 +118,7 @@ std::string computeMessageHash(
         std::chrono::system_clock::time_point timestamp,
         std::chrono::system_clock::time_point expiry,
         const user_pubkey_t& pubkey,
+        namespace_id ns,
         std::string_view data);
 
 struct OnionRequestMetadata {

httpserver/service_node.cpp

@@ -1,6 +1,5 @@
 #include "service_node.h"
 
-#include "Database.hpp"
 #include "http.h"
 #include "omq_server.h"
 #include "oxen_logger.h"
@@ -182,8 +181,8 @@ static block_update parse_swarm_update(const std::string& response_body) {
             }
         }
 
-        if (missing_aux_pks
-            > MISSING_PUBKEY_THRESHOLD::num * total / MISSING_PUBKEY_THRESHOLD::den) {
+        if (missing_aux_pks >
+            MISSING_PUBKEY_THRESHOLD::num * total / MISSING_PUBKEY_THRESHOLD::den) {
             OXEN_LOG(
                     warn,
                     "Missing ed25519/x25519 pubkeys for {}/{} service nodes; "
@@ -388,6 +387,10 @@ void ServiceNode::record_onion_request() {
     all_stats_.bump_onion_requests();
 }
 
+void ServiceNode::record_retrieve_request() {
+    all_stats_.bump_retrieve_requests();
+}
+
 bool ServiceNode::process_store(message msg, bool* new_msg) {
     std::lock_guard guard{sn_mutex_};
 
@@ -456,8 +459,8 @@ static SnodeStatus derive_snode_status(const block_update& bu, const sn_record&
         return SnodeStatus::ACTIVE;
     }
 
-    if (std::find(bu.decommissioned_nodes.begin(), bu.decommissioned_nodes.end(), our_address)
-        != bu.decommissioned_nodes.end()) {
+    if (std::find(bu.decommissioned_nodes.begin(), bu.decommissioned_nodes.end(), our_address) !=
+        bu.decommissioned_nodes.end()) {
         return SnodeStatus::DECOMMISSIONED;
     }
 
@@ -539,7 +542,7 @@ void ServiceNode::on_swarm_update(block_update&& bu) {
     swarm_->update_state(bu.swarms, bu.decommissioned_nodes, events, true);
 
     if (!events.new_snodes.empty()) {
-        relay_messages(get_all_messages(), events.new_snodes);
+        relay_messages(db_->retrieve_all(), events.new_snodes);
     }
 
     if (!events.new_swarms.empty()) {
@@ -551,7 +554,9 @@ void ServiceNode::on_swarm_update(block_update&& bu) {
         bootstrap_swarms();
     }
 
-    initiate_peer_test();
+    // Peer testing has never worked reliably (there are lots of race conditions around when blocks
+    // change) and isn't enforce on the network, so just disable initiating testing for now:
+    // initiate_peer_test();
 }
 
 void ServiceNode::update_swarms() {
@@ -610,9 +615,9 @@ void ServiceNode::update_swarms() {
                             omq_server_.oxend_request(
                                     "rpc.get_block_hash",
                                     [this, h](bool success, std::vector<std::string> data) {
-                                        if (!(success && data.size() == 2 && data[0] == "200"
-                                              && data[1].size() == 66 && data[1].front() == '"'
-                                              && data[1].back() == '"'))
+                                        if (!(success && data.size() == 2 && data[0] == "200" &&
+                                              data[1].size() == 66 && data[1].front() == '"' &&
+                                              data[1].back() == '"'))
                                             return;
                                         std::string_view hash{
                                                 data[1].data() + 1, data[1].size() - 2};
@@ -640,9 +645,9 @@ void ServiceNode::update_swarms() {
                         // nodes: but currently we still need this to deal with the lag).
 
                         auto [missing, total] = count_missing_data(bu);
-                        if (total >= (oxen::is_mainnet ? 100 : 10)
-                            && missing <= MISSING_PUBKEY_THRESHOLD::num * total
-                                                  / MISSING_PUBKEY_THRESHOLD::den) {
+                        if (total >= (oxen::is_mainnet ? 100 : 10) &&
+                            missing <= MISSING_PUBKEY_THRESHOLD::num * total /
+                                               MISSING_PUBKEY_THRESHOLD::den) {
                             OXEN_LOG(
                                     info,
                                     "Initialized from oxend with {}/{} SN records",
@@ -1154,7 +1159,7 @@ void ServiceNode::bootstrap_swarms(const std::vector<swarm_id_t>& swarms) const
     std::unordered_map<user_pubkey_t, swarm_id_t> pk_swarm_cache;
     std::unordered_map<swarm_id_t, std::vector<message>> to_relay;
 
-    std::vector<message> all_entries = get_all_messages();
+    std::vector<message> all_entries = db_->retrieve_all();
     OXEN_LOG(debug, "We have {} messages", all_entries.size());
     for (auto& entry : all_entries) {
         if (!entry.pubkey) {
@@ -1202,39 +1207,6 @@ void ServiceNode::relay_messages(
             relay_data_reliable(batch, sn);
 }
 
-std::vector<message> ServiceNode::retrieve(
-        const user_pubkey_t& pubkey, const std::string& last_hash) {
-    all_stats_.bump_retrieve_requests();
-    return db_->retrieve(pubkey, last_hash, CLIENT_RETRIEVE_MESSAGE_LIMIT);
-}
-
-std::optional<std::vector<std::string>> ServiceNode::delete_all_messages(
-        const user_pubkey_t& pubkey) {
-    return db_->delete_all(pubkey);
-}
-
-std::optional<std::vector<std::string>> ServiceNode::delete_messages(
-        const user_pubkey_t& pubkey, const std::vector<std::string>& msg_hashes) {
-    return db_->delete_by_hash(pubkey, msg_hashes);
-}
-
-std::optional<std::vector<std::string>> ServiceNode::delete_messages_before(
-        const user_pubkey_t& pubkey, std::chrono::system_clock::time_point timestamp) {
-    return db_->delete_by_timestamp(pubkey, timestamp);
-}
-
-std::optional<std::vector<std::string>> ServiceNode::update_messages_expiry(
-        const user_pubkey_t& pubkey,
-        const std::vector<std::string>& msg_hashes,
-        std::chrono::system_clock::time_point new_exp) {
-    return db_->update_expiry(pubkey, msg_hashes, new_exp);
-}
-
-std::optional<std::vector<std::string>> ServiceNode::update_all_expiries(
-        const user_pubkey_t& pubkey, std::chrono::system_clock::time_point new_exp) {
-    return db_->update_all_expiries(pubkey, new_exp);
-}
-
 void to_json(nlohmann::json& j, const test_result& val) {
     j["timestamp"] = std::chrono::duration<double>(val.timestamp.time_since_epoch()).count();
     j["result"] = to_str(val.result);
@@ -1335,11 +1307,6 @@ std::string ServiceNode::get_status_line() const {
     return s.str();
 }
 
-std::vector<message> ServiceNode::get_all_messages() const {
-    OXEN_LOG(trace, "Get all messages");
-    return db_->retrieve_all();
-}
-
 void ServiceNode::process_push_batch(const std::string& blob) {
     std::lock_guard guard(sn_mutex_);
 

httpserver/service_node.h

@@ -41,6 +41,10 @@ using hf_revision = std::pair<int, int>;
 // The earliest hardfork *this* version of storage server will work on:
 inline constexpr hf_revision STORAGE_SERVER_HARDFORK = {18, 1};
 
+// The hardfork at which we require authentication for (almost) all retrieval.  (Message namespace
+// -10 is temporarily exempt for closed group backwards support).
+inline constexpr hf_revision HARDFORK_RETRIEVE_AUTH = {19, 0};
+
 class OxenmqServer;
 struct OnionRequestMetadata;
 class Swarm;
@@ -161,16 +165,20 @@ class ServiceNode {
             const std::filesystem::path& db_location,
             bool force_start);
 
+    Database& get_db() { return *db_; }
+    const Database& get_db() const { return *db_; }
+
     // Return info about this node as it is advertised to other nodes
     const sn_record& own_address() { return our_address_; }
 
     // Record the time of our last being tested over omq/https
     void update_last_ping(ReachType type);
 
-    // These two are only needed because we store stats in Service Node,
+    // These three are only needed because we store stats in Service Node,
     // might move it out later
     void record_proxy_request();
     void record_onion_request();
+    void record_retrieve_request();
 
     /// Sends an onion request to the next SS
     void send_onion_to_sn(
@@ -217,42 +225,6 @@ class ServiceNode {
 
     std::vector<sn_record> get_swarm_peers();
 
-    std::vector<message> get_all_messages() const;
-
-    /// return all messages for a particular PK
-    std::vector<message> retrieve(const user_pubkey_t& pubkey, const std::string& last_hash);
-
-    /// Deletes all messages belonging to a pubkey; returns the deleted hashes
-    std::optional<std::vector<std::string>> delete_all_messages(const user_pubkey_t& pubkey);
-
-    /// Delete messages owned by the given pubkey having the given hashes.  Returns the hashes
-    /// of any delete messages on success (including the case where no messages are deleted),
-    /// nullopt on query failure.
-    std::optional<std::vector<std::string>> delete_messages(
-            const user_pubkey_t& pubkey, const std::vector<std::string>& msg_hashes);
-
-    /// Deletes all messages owned by the given pubkey with a timestamp <= `timestamp`.  Returns
-    /// the hashes of any deleted messages (including the case where no messages are deleted),
-    /// nullopt on query failure.
-    std::optional<std::vector<std::string>> delete_messages_before(
-            const user_pubkey_t& pubkey, std::chrono::system_clock::time_point timestamp);
-
-    /// Shortens the expiry time of the given messages owned by the given pubkey.  Expiries can
-    /// only be shortened (i.e. brought closer to now), not extended into the future.  Returns a
-    /// vector of [msgid, newexpiry] pairs indicating the new expiry of any messages found (note
-    /// that the new expiry may not have been updated if it was already shorter than the
-    /// requested time).
-    std::optional<std::vector<std::string>> update_messages_expiry(
-            const user_pubkey_t& pubkey,
-            const std::vector<std::string>& msg_hashes,
-            std::chrono::system_clock::time_point new_exp);
-
-    /// Shortens the expiry time of all messages owned by the given pubkey.  Expiries can only
-    /// be shortened (i.e. brought closer to now), not extended into the future.  Returns a
-    /// vector of [msg, newexpiry] for all messages, whether the expiry is updated or not.
-    std::optional<std::vector<std::string>> update_all_expiries(
-            const user_pubkey_t& pubkey, std::chrono::system_clock::time_point new_exp);
-
     // Stats for session clients that want to know the version number
     std::string get_stats_for_session_client() const;
 

httpserver/stats.cpp

@@ -51,8 +51,8 @@ std::pair<std::chrono::steady_clock::duration, period_stats> all_stats_t::get_re
     auto& [window, stats] = result;
 
     std::lock_guard lock{prev_stats_mutex};
-    window = std::chrono::steady_clock::now()
-           - (previous_stats.empty() ? last_rotate : previous_stats.front().first);
+    window = std::chrono::steady_clock::now() -
+             (previous_stats.empty() ? last_rotate : previous_stats.front().first);
 
     stats = {
             current_client_store_requests,

httpserver/swarm.cpp

@@ -341,8 +341,8 @@ std::pair<int, int> count_missing_data(const block_update& bu) {
     for (auto& swarm : bu.swarms) {
         for (auto& snode : swarm.snodes) {
             total++;
-            if (snode.ip.empty() || snode.ip == "0.0.0.0" || !snode.port || !snode.omq_port
-                || !snode.pubkey_ed25519 || !snode.pubkey_x25519) {
+            if (snode.ip.empty() || snode.ip == "0.0.0.0" || !snode.port || !snode.omq_port ||
+                !snode.pubkey_ed25519 || !snode.pubkey_x25519) {
                 OXEN_LOG(
                         warn,
                         "well wtf {} {} {} {} {}",

network-tests/README.md

@@ -5,15 +5,9 @@ testnet.
 
 Usage:
 
-- install the pyoxenmq Python module from the pyoxenmq submodule.  You can do this locally via
-  `python3 setup.py build` and then symlink the .so into the storage server tests/ directory (the
-  file and directory names in this example are Python version and system dependent):
-
-    ln -s pyoxenmq/build/lib.linux-x86_64-3.9/pyoxenmq.cpython-39-x86_64-linux-gnu.so .
-
-  Alternatively, rather than symlinking, simply install as a user with:
-
-    python3 setup.py install --user
+- install the [https://ci.oxen.rocks/oxen-io/oxen-pyoxenmq](oxenmq Python module).  You can build it
+  from source, or alternatively grab the python3-oxenmq deb package from our deb repo
+  (https://deb.oxen.io)u.
 
 - Run `py.test-3` to run the test suite.  (You likely need to install python3-pytest and
   python3-nacl, if not already installed).

network-tests/conftest.py

@@ -1,6 +1,6 @@
 
 import pytest
-import pyoxenmq
+from oxenmq import OxenMQ, Address
 import json
 import random
 
@@ -10,15 +10,15 @@ def pytest_addoption(parser):
 
 @pytest.fixture(scope="module")
 def omq():
-    omq = pyoxenmq.OxenMQ()
+    omq = OxenMQ()
     omq.start()
     return omq
 
 
 @pytest.fixture(scope="module")
 def sns(omq):
-    remote = omq.connect_remote("curve://public.loki.foundation:38161/80adaead94db3b0402a6057869bdbe63204a28e93589fd95a035480ed6c03b45")
-    x = omq.request(remote, "rpc.get_service_nodes")
+    remote = omq.connect_remote(Address("curve://public.loki.foundation:38161/80adaead94db3b0402a6057869bdbe63204a28e93589fd95a035480ed6c03b45"))
+    x = omq.request_future(remote, "rpc.get_service_nodes", b'{"active_only": true}').get()
     assert(len(x) == 2 and x[0] == b'200')
     return json.loads(x[1])
 
@@ -26,7 +26,7 @@ def sns(omq):
 @pytest.fixture(scope="module")
 def random_sn(omq, sns):
     sn = random.choice(sns['service_node_states'])
-    addr = "curve://{}:{}/{}".format(sn['public_ip'], sn['storage_lmq_port'], sn['pubkey_x25519'])
+    addr = Address(sn['public_ip'], sn['storage_lmq_port'], bytes.fromhex(sn['pubkey_x25519']))
     conn = omq.connect_remote(addr)
     return conn
 

network-tests/pyoxenmq

@@ -1 +0,0 @@
-Subproject commit b5f3ff6132a0ca3e7a6d342ef33588dd3b0e1665

network-tests/ss.py

@@ -49,7 +49,7 @@ def delete_before(sk, *, ago=120, timestamp=None):
 
 def get_swarm(omq, conn, sk, netid=5):
     pubkey = "{:02x}".format(netid) + (sk.verify_key if isinstance(sk, SigningKey) else sk.public_key).encode().hex()
-    r = omq.request(conn, "storage.get_swarm", [json.dumps({"pubkey": pubkey}).encode()])
+    r = omq.request_future(conn, "storage.get_swarm", [json.dumps({"pubkey": pubkey}).encode()]).get()
     assert(len(r) == 1)
     return json.loads(r[0])
 

network-tests/test_deletes.py

@@ -1,4 +1,4 @@
-import pyoxenmq
+from util import sn_address
 import ss
 import time
 import base64
@@ -11,8 +11,7 @@ import nacl.exceptions
 def test_delete_all(omq, random_sn, sk, exclude):
     swarm = ss.get_swarm(omq, random_sn, sk)
     sns = ss.random_swarm_members(swarm, 2, exclude)
-    conns = [omq.connect_remote("curve://{}:{}/{}".format(sn['ip'], sn['port_omq'], sn['pubkey_x25519']))
-            for sn in sns]
+    conns = [omq.connect_remote(sn_address(sn)) for sn in sns]
 
     msgs = ss.store_n(omq, conns[0], sk, b"omg123", 5)
 
@@ -27,7 +26,7 @@ def test_delete_all(omq, random_sn, sk, exclude):
             "signature": sig
     }).encode()
 
-    resp = omq.request(conns[1], 'storage.delete_all', [params])
+    resp = omq.request_future(conns[1], 'storage.delete_all', [params]).get()
 
     assert len(resp) == 1
     r = json.loads(resp[0])
@@ -43,16 +42,22 @@ def test_delete_all(omq, random_sn, sk, exclude):
         edpk = VerifyKey(k, encoder=HexEncoder)
         edpk.verify(expected_signed, base64.b64decode(v['signature']))
 
-    r = json.loads(omq.request(conns[0], 'storage.retrieve',
-        [json.dumps({ "pubkey": my_ss_id }).encode()]
-        )[0])
+    r = omq.request_future(conns[0], 'storage.retrieve',
+        [json.dumps({
+            "pubkey": my_ss_id,
+            "timestamp": ts,
+            "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode()
+            }).encode()]
+        ).get()
+    assert len(r) == 1
+    r = json.loads(r[0])
     assert not r['messages']
 
 
 def test_stale_delete_all(omq, random_sn, sk, exclude):
     swarm = ss.get_swarm(omq, random_sn, sk)
     sn = ss.random_swarm_members(swarm, 2, exclude)[0]
-    conn = omq.connect_remote("curve://{}:{}/{}".format(sn['ip'], sn['port_omq'], sn['pubkey_x25519']))
+    conn = omq.connect_remote(sn_address(sn))
 
     msgs = ss.store_n(omq, conn, sk, b"omg123", 5)
 
@@ -67,23 +72,23 @@ def test_stale_delete_all(omq, random_sn, sk, exclude):
             "signature": sig
     }
 
-    resp = omq.request(conn, 'storage.delete_all', [json.dumps(params).encode()])
-    assert resp == [b'406', b'delete_all timestamp too far from current time']
+    resp_too_old = omq.request_future(conn, 'storage.delete_all', [json.dumps(params).encode()])
 
     ts = int((time.time() + 120) * 1000)
     to_sign = "delete_all{}".format(ts).encode()
     sig = sk.sign(to_sign, encoder=Base64Encoder).signature.decode()
     params["signature"] = sig
 
-    resp = omq.request(conn, 'storage.delete_all', [json.dumps(params).encode()])
-    assert resp == [b'406', b'delete_all timestamp too far from current time']
+    resp_too_new = omq.request_future(conn, 'storage.delete_all', [json.dumps(params).encode()])
+
+    assert resp_too_old.get() == [b'406', b'delete_all timestamp too far from current time']
+    assert resp_too_new.get() == [b'406', b'delete_all timestamp too far from current time']
 
 
 def test_delete(omq, random_sn, sk, exclude):
     swarm = ss.get_swarm(omq, random_sn, sk, netid=2)
     sns = ss.random_swarm_members(swarm, 2, exclude)
-    conns = [omq.connect_remote("curve://{}:{}/{}".format(sn['ip'], sn['port_omq'], sn['pubkey_x25519']))
-            for sn in sns]
+    conns = [omq.connect_remote(sn_address(sn)) for sn in sns]
 
     msgs = ss.store_n(omq, conns[0], sk, b"omg123", 5, netid=2)
 
@@ -101,7 +106,7 @@ def test_delete(omq, random_sn, sk, exclude):
             "signature": sig
     }).encode()
 
-    resp = omq.request(conns[1], 'storage.delete', [params])
+    resp = omq.request_future(conns[1], 'storage.delete', [params]).get()
 
     assert len(resp) == 1
     r = json.loads(resp[0])
@@ -120,17 +125,22 @@ def test_delete(omq, random_sn, sk, exclude):
             print("Bad signature from swarm member {}".format(k))
             raise e
 
-    r = json.loads(omq.request(conns[0], 'storage.retrieve',
-        [json.dumps({ "pubkey": my_ss_id }).encode()]
-        )[0])
+    r = omq.request_future(conns[0], 'storage.retrieve',
+        [json.dumps({
+            "pubkey": my_ss_id,
+            "timestamp": ts,
+            "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode()
+            }).encode()]
+        ).get()
+    assert len(r) == 1
+    r = json.loads(r[0])
     assert len(r['messages']) == 3
 
 
 def test_delete_before(omq, random_sn, sk, exclude):
     swarm = ss.get_swarm(omq, random_sn, sk)
     sns = ss.random_swarm_members(swarm, 2, exclude)
-    conns = [omq.connect_remote("curve://{}:{}/{}".format(sn['ip'], sn['port_omq'], sn['pubkey_x25519']))
-            for sn in sns]
+    conns = [omq.connect_remote(sn_address(sn)) for sn in sns]
 
     msgs = ss.store_n(omq, conns[0], sk, b"omg123", 10)
 
@@ -151,7 +161,7 @@ def test_delete_before(omq, random_sn, sk, exclude):
             "signature": sig
     }).encode()
 
-    resp = omq.request(conns[1], 'storage.delete_before', [params])
+    resp = omq.request_future(conns[1], 'storage.delete_before', [params]).get()
 
     assert len(resp) == 1
     r = json.loads(resp[0])
@@ -169,9 +179,15 @@ def test_delete_before(omq, random_sn, sk, exclude):
             print("Bad signature from swarm member {}".format(k))
             raise e
 
-    r = json.loads(omq.request(conns[0], 'storage.retrieve',
-        [json.dumps({ "pubkey": my_ss_id }).encode()]
-        )[0])
+    r = omq.request_future(conns[0], 'storage.retrieve',
+        [json.dumps({
+            "pubkey": my_ss_id,
+            "timestamp": ts,
+            "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode()
+            }).encode()]
+        ).get()
+    assert len(r) == 1
+    r = json.loads(r[0])
     assert len(r['messages']) == 8
 
 
@@ -185,7 +201,7 @@ def test_delete_before(omq, random_sn, sk, exclude):
             "signature": sig
     }).encode()
 
-    resp = omq.request(conns[0], 'storage.delete_before', [params])
+    resp = omq.request_future(conns[0], 'storage.delete_before', [params]).get()
 
     assert len(resp) == 1
     r = json.loads(resp[0])
@@ -203,9 +219,15 @@ def test_delete_before(omq, random_sn, sk, exclude):
             print("Bad signature from swarm member {}".format(k))
             raise e
 
-    r = json.loads(omq.request(conns[0], 'storage.retrieve',
-        [json.dumps({ "pubkey": my_ss_id }).encode()]
-        )[0])
+    r = omq.request_future(conns[0], 'storage.retrieve',
+        [json.dumps({
+            "pubkey": my_ss_id,
+            "timestamp": ts,
+            "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode()
+            }).encode()]
+        ).get()
+    assert len(r) == 1
+    r = json.loads(r[0])
     assert len(r['messages']) == 8
 
 
@@ -221,7 +243,7 @@ def test_delete_before(omq, random_sn, sk, exclude):
             "signature": sig
     }).encode()
 
-    resp = omq.request(conns[0], 'storage.delete_before', [params])
+    resp = omq.request_future(conns[0], 'storage.delete_before', [params]).get()
 
     assert len(resp) == 1
     r = json.loads(resp[0])
@@ -239,9 +261,15 @@ def test_delete_before(omq, random_sn, sk, exclude):
             print("Bad signature from swarm member {}".format(k))
             raise e
 
-    r = json.loads(omq.request(conns[0], 'storage.retrieve',
-        [json.dumps({ "pubkey": my_ss_id }).encode()]
-        )[0])
+    r = omq.request_future(conns[0], 'storage.retrieve',
+        [json.dumps({
+            "pubkey": my_ss_id,
+            "timestamp": ts,
+            "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode()
+            }).encode()]
+        ).get()
+    assert len(r) == 1
+    r = json.loads(r[0])
     assert len(r['messages']) == 1
 
 
@@ -257,7 +285,7 @@ def test_delete_before(omq, random_sn, sk, exclude):
             "signature": sig
     }).encode()
 
-    resp = omq.request(conns[1], 'storage.delete_before', [params])
+    resp = omq.request_future(conns[1], 'storage.delete_before', [params]).get()
 
     assert len(resp) == 1
     r = json.loads(resp[0])
@@ -275,7 +303,13 @@ def test_delete_before(omq, random_sn, sk, exclude):
             print("Bad signature from swarm member {}".format(k))
             raise e
 
-    r = json.loads(omq.request(conns[1], 'storage.retrieve',
-        [json.dumps({ "pubkey": my_ss_id }).encode()]
-        )[0])
+    r = omq.request_future(conns[1], 'storage.retrieve',
+        [json.dumps({
+            "pubkey": my_ss_id,
+            "timestamp": ts,
+            "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode()
+            }).encode()]
+        ).get()
+    assert len(r) == 1
+    r = json.loads(r[0])
     assert not r['messages']

network-tests/test_expire.py

@@ -1,5 +1,5 @@
-import pyoxenmq
 import ss
+from util import sn_address
 import time
 import base64
 import json
@@ -11,8 +11,7 @@ import nacl.exceptions
 def test_expire_all(omq, random_sn, sk, exclude):
     swarm = ss.get_swarm(omq, random_sn, sk)
     sns = ss.random_swarm_members(swarm, 2, exclude)
-    conns = [omq.connect_remote("curve://{}:{}/{}".format(sn['ip'], sn['port_omq'], sn['pubkey_x25519']))
-            for sn in sns]
+    conns = [omq.connect_remote(sn_address(sn)) for sn in sns]
 
     msgs = ss.store_n(omq, conns[0], sk, b"omg123", 5)
 
@@ -27,7 +26,7 @@ def test_expire_all(omq, random_sn, sk, exclude):
             "signature": sig
     }).encode()
 
-    resp = omq.request(conns[1], 'storage.expire_all', [params])
+    resp = omq.request_future(conns[1], 'storage.expire_all', [params]).get()
 
     assert len(resp) == 1
     r = json.loads(resp[0])
@@ -46,9 +45,15 @@ def test_expire_all(omq, random_sn, sk, exclude):
         edpk = VerifyKey(k, encoder=HexEncoder)
         edpk.verify(expected_signed, base64.b64decode(v['signature']))
 
-    r = json.loads(omq.request(conns[0], 'storage.retrieve',
-        [json.dumps({ "pubkey": my_ss_id }).encode()]
-        )[0])
+    r = omq.request_future(conns[0], 'storage.retrieve',
+        [json.dumps({
+            "pubkey": my_ss_id,
+            "timestamp": ts,
+            "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode()
+            }).encode()]
+        ).get()
+    assert len(r) == 1
+    r = json.loads(r[0])
     assert len(r['messages']) == 5
 
     assert r['messages'][0]['expiration'] == ts
@@ -61,7 +66,7 @@ def test_expire_all(omq, random_sn, sk, exclude):
 def test_stale_expire_all(omq, random_sn, sk, exclude):
     swarm = ss.get_swarm(omq, random_sn, sk)
     sn = ss.random_swarm_members(swarm, 2, exclude)[0]
-    conn = omq.connect_remote("curve://{}:{}/{}".format(sn['ip'], sn['port_omq'], sn['pubkey_x25519']))
+    conn = omq.connect_remote(sn_address(sn))
 
     msgs = ss.store_n(omq, conn, sk, b"omg123", 5)
 
@@ -76,15 +81,14 @@ def test_stale_expire_all(omq, random_sn, sk, exclude):
             "signature": sig
     }
 
-    resp = omq.request(conn, 'storage.expire_all', [json.dumps(params).encode()])
+    resp = omq.request_future(conn, 'storage.expire_all', [json.dumps(params).encode()]).get()
     assert resp == [b'406', b'expire_all timestamp should be >= current time']
 
 
 def test_expire(omq, random_sn, sk, exclude):
     swarm = ss.get_swarm(omq, random_sn, sk)
     sns = ss.random_swarm_members(swarm, 2, exclude)
-    conns = [omq.connect_remote("curve://{}:{}/{}".format(sn['ip'], sn['port_omq'], sn['pubkey_x25519']))
-            for sn in sns]
+    conns = [omq.connect_remote(sn_address(sn)) for sn in sns]
 
     msgs = ss.store_n(omq, conns[0], sk, b"omg123", 10)
 
@@ -104,7 +108,7 @@ def test_expire(omq, random_sn, sk, exclude):
             "signature": sig
     }).encode()
 
-    resp = omq.request(conns[1], 'storage.expire', [params])
+    resp = omq.request_future(conns[1], 'storage.expire', [params]).get()
 
     assert len(resp) == 1
     r = json.loads(resp[0])
@@ -122,9 +126,15 @@ def test_expire(omq, random_sn, sk, exclude):
             print("Bad signature from swarm member {}".format(k))
             raise e
 
-    r = json.loads(omq.request(conns[0], 'storage.retrieve',
-        [json.dumps({ "pubkey": my_ss_id }).encode()]
-        )[0])
+    r = omq.request_future(conns[0], 'storage.retrieve',
+        [json.dumps({
+            "pubkey": my_ss_id,
+            "timestamp": ts,
+            "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode()
+            }).encode()]
+        ).get()
+    assert len(r) == 1
+    r = json.loads(r[0])
     assert len(r['messages']) == 10
 
     for i in range(10):

network-tests/test_msg_ns.py

@@ -0,0 +1,166 @@
+from util import sn_address
+import ss
+import time
+import base64
+import json
+import secrets
+from nacl.encoding import HexEncoder, Base64Encoder
+from nacl.hash import blake2b
+from nacl.signing import SigningKey, VerifyKey
+
+def test_store_ns(omq, random_sn, sk, exclude):
+    swarm = ss.get_swarm(omq, random_sn, sk)
+
+    sn = ss.random_swarm_members(swarm, 1, exclude)[0]
+    conn = omq.connect_remote(sn_address(sn))
+
+    ts = int(time.time() * 1000)
+    ttl = 86400000
+    exp = ts + ttl
+    # Store a message (publicly depositable namespace, divisible by 10)
+    spub = omq.request_future(conn, 'storage.store', [json.dumps({
+        "pubkey": '05' + sk.verify_key.encode().hex(),
+        "timestamp": ts,
+        "ttl": ttl,
+        "namespace": 40,
+        "data": base64.b64encode("abc 123".encode()).decode()}).encode()])
+
+    # Store a message for myself in a private namespace (not divisible by 10)
+    spriv = omq.request_future(conn, 'storage.store', [json.dumps({
+        "pubkey": '05' + sk.verify_key.encode().hex(),
+        "timestamp": ts,
+        "ttl": ttl,
+        "namespace": -42,
+        "data": base64.b64encode("abc 123".encode()).decode(),
+        "signature": sk.sign(f"store-42{ts}".encode(), encoder=Base64Encoder).signature.decode()}).encode()])
+
+
+    
+    spub = json.loads(spub.get()[0])
+
+    hash = blake2b("{}{}".format(ts, exp).encode() + b'\x05' + sk.verify_key.encode() + b'40' + b'abc 123',
+            encoder=Base64Encoder).decode().rstrip('=')
+
+    assert len(spub["swarm"]) == len(swarm['snodes'])
+    edkeys = {x['pubkey_ed25519'] for x in swarm['snodes']}
+    for k, v in spub['swarm'].items():
+        assert k in edkeys
+        assert hash == v['hash']
+
+        edpk = VerifyKey(k, encoder=HexEncoder)
+        edpk.verify(v['hash'].encode(), base64.b64decode(v['signature']))
+
+    # NB: assumes the test machine is reasonably time synced
+    assert(ts - 30000 <= spub['t'] <= ts + 30000)
+
+
+    spriv = json.loads(spriv.get()[0])
+    hash = blake2b("{}{}".format(ts, exp).encode() + b'\x05' + sk.verify_key.encode() + b'-42' + b'abc 123',
+            encoder=Base64Encoder).decode().rstrip('=')
+
+    assert len(spriv["swarm"]) == len(swarm['snodes'])
+    edkeys = {x['pubkey_ed25519'] for x in swarm['snodes']}
+    for k, v in spriv['swarm'].items():
+        assert k in edkeys
+        assert hash == v['hash']
+
+        edpk = VerifyKey(k, encoder=HexEncoder)
+        edpk.verify(v['hash'].encode(), base64.b64decode(v['signature']))
+
+    # NB: assumes the test machine is reasonably time synced
+    assert(ts - 30000 <= spriv['t'] <= ts + 30000)
+
+
+def test_legacy_closed_ns(omq, random_sn, sk, exclude):
+    # For legacy closed groups the secret key is generated but then immediately discarded; it's only
+    # used to generate a primary key storage address:
+
+    swarm = ss.get_swarm(omq, random_sn, sk)
+
+    sn = ss.random_swarm_members(swarm, 1, exclude)[0]
+    conn = omq.connect_remote(sn_address(sn))
+
+    ts = int(time.time() * 1000)
+    ttl = 86400000
+    exp = ts + ttl
+
+    # namespace -10 is a special, no-auth namespace for legacy closed group messages.
+    sclosed = omq.request_future(conn, 'storage.store', [json.dumps({
+        "pubkey": '05' + sk.verify_key.encode().hex(),
+        "timestamp": ts,
+        "ttl": ttl,
+        "namespace": -10,
+        "data": base64.b64encode("blah blah".encode()).decode()})])
+
+    sclosed = json.loads(sclosed.get()[0])
+    hash = blake2b("{}{}".format(ts, exp).encode() + b'\x05' + sk.verify_key.encode() + b'-10' + b'blah blah',
+            encoder=Base64Encoder).decode().rstrip('=')
+
+    assert len(sclosed["swarm"]) == len(swarm['snodes'])
+    edkeys = {x['pubkey_ed25519'] for x in swarm['snodes']}
+    for k, v in sclosed['swarm'].items():
+        assert k in edkeys
+        assert hash == v['hash']
+
+        edpk = VerifyKey(k, encoder=HexEncoder)
+        edpk.verify(v['hash'].encode(), base64.b64decode(v['signature']))
+
+    # NB: assumes the test machine is reasonably time synced
+    assert(ts - 30000 <= sclosed['t'] <= ts + 30000)
+
+    # Now retrieve it: this is the only namespace we can access without authentication
+    r = omq.request_future(conn, 'storage.retrieve', [json.dumps({
+        "pubkey": '05' + sk.verify_key.encode().hex(),
+        "namespace": -10,
+    }).encode()])
+
+    r = r.get()
+    assert len(r) == 1
+    r = json.loads(r[0])
+
+    assert len(r['messages']) == 1
+    msg = r['messages'][0]
+    assert base64.b64decode(msg['data']) == b'blah blah'
+    assert msg['timestamp'] == ts
+    assert msg['expiration'] == exp
+    assert msg['hash'] == hash
+
+
+def test_store_invalid_ns(omq, random_sn, sk, exclude):
+    swarm = ss.get_swarm(omq, random_sn, sk)
+
+    sn = ss.random_swarm_members(swarm, 1, exclude)[0]
+    conn = omq.connect_remote(sn_address(sn))
+
+    ts = int(time.time() * 1000)
+    ttl = 86400000
+    exp = ts + ttl
+    # Attempt to store a message without authentication in a non-public (% 10 != 0) namespace:
+    s42 = omq.request_future(conn, 'storage.store', [json.dumps({
+        "pubkey": '05' + sk.verify_key.encode().hex(),
+        "timestamp": ts,
+        "ttl": ttl,
+        "namespace": 42,
+        "data": base64.b64encode("abc 123".encode()).decode()}).encode()])
+
+    # Attempt to store a message in a too-big/too-small namespace:
+    s32k = omq.request_future(conn, 'storage.store', [json.dumps({
+        "pubkey": '05' + sk.verify_key.encode().hex(),
+        "timestamp": ts,
+        "ttl": ttl,
+        "namespace": 32768,
+        "data": base64.b64encode("abc 123".encode()).decode()}).encode()])
+
+    # Bad signature:
+    dude_sk = SigningKey.generate()
+    sdude = omq.request_future(conn, 'storage.store', [json.dumps({
+        "pubkey": '05' + sk.verify_key.encode().hex(),
+        "timestamp": ts,
+        "ttl": ttl,
+        "namespace": -32123,
+        "signature": dude_sk.sign(f"store-32123{ts}".encode(), encoder=Base64Encoder).signature.decode(),
+        "data": base64.b64encode("abc 123".encode()).decode()}).encode()])
+
+    assert s42.get() == [b'401', b'store: signature required to store to namespace 42']
+    assert s32k.get() == [b'400', b"invalid request: Invalid value given for 'namespace': value out of range"]
+    assert sdude.get() == [b'401', b"store signature verification failed"]

network-tests/test_session_auth.py

@@ -1,5 +1,5 @@
-import pyoxenmq
 import ss
+from util import sn_address
 import time
 import base64
 import json
@@ -21,7 +21,7 @@ def test_session_auth(omq, random_sn, sk, exclude):
 
     swarm = ss.get_swarm(omq, random_sn, xsk)
     sn = ss.random_swarm_members(swarm, 1, exclude)[0]
-    conn = omq.connect_remote("curve://{}:{}/{}".format(sn['ip'], sn['port_omq'], sn['pubkey_x25519']))
+    conn = omq.connect_remote(sn_address(sn))
 
     msgs = ss.store_n(omq, conn, xsk, b"omg123", 5)
 
@@ -36,15 +36,22 @@ def test_session_auth(omq, random_sn, sk, exclude):
             "signature": sig
     }
 
-    resp = omq.request(conn, 'storage.delete_all', [json.dumps(params).encode()])
+    resp = omq.request_future(conn, 'storage.delete_all', [json.dumps(params).encode()]).get()
 
     # Expect this to fail because we didn't pass the Ed25519 key
     assert resp == [b'401', b'delete_all signature verification failed']
 
     # Make sure nothing was actually deleted:
-    r = json.loads(omq.request(conn, 'storage.retrieve',
-        [json.dumps({ "pubkey": my_ss_id }).encode()]
-        )[0])
+    r = omq.request_future(conn, 'storage.retrieve',
+        [json.dumps({
+            "pubkey": my_ss_id,
+            "pubkey_ed25519": sk.verify_key.encode().hex(),
+            "timestamp": ts,
+            "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode(),
+            }).encode()]
+        ).get()
+    assert len(r) == 1
+    r = json.loads(r[0])
     assert len(r['messages']) == 5
 
     # Try signing with some *other* ed25519 key, which should be detected as not corresponding to
@@ -53,20 +60,27 @@ def test_session_auth(omq, random_sn, sk, exclude):
     fake_sig = fake_sk.sign(to_sign, encoder=Base64Encoder).signature.decode()
     params['pubkey_ed25519'] = fake_sk.verify_key.encode().hex()
     params['signature'] = fake_sig
-    resp = omq.request(conn, 'storage.delete_all', [json.dumps(params).encode()])
+    resp = omq.request_future(conn, 'storage.delete_all', [json.dumps(params).encode()]).get()
 
     assert resp == [b'401', b'delete_all signature verification failed']
 
     # Make sure nothing was actually deleted:
-    r = json.loads(omq.request(conn, 'storage.retrieve',
-        [json.dumps({ "pubkey": my_ss_id }).encode()]
-        )[0])
+    r = omq.request_future(conn, 'storage.retrieve',
+        [json.dumps({
+            "pubkey": my_ss_id,
+            "pubkey_ed25519": sk.verify_key.encode().hex(),
+            "timestamp": ts,
+            "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode(),
+            }).encode()]
+        ).get()
+    assert len(r) == 1
+    r = json.loads(r[0])
     assert len(r['messages']) == 5
 
     # Now send along the correct ed pubkey to make it work
     params['pubkey_ed25519'] = sk.verify_key.encode().hex()
     params['signature'] = sig
-    resp = omq.request(conn, 'storage.delete_all', [json.dumps(params).encode()])
+    resp = omq.request_future(conn, 'storage.delete_all', [json.dumps(params).encode()]).get()
 
     assert len(resp) == 1
     r = json.loads(resp[0])
@@ -81,9 +95,16 @@ def test_session_auth(omq, random_sn, sk, exclude):
 
 
     # Verify deletion
-    r = json.loads(omq.request(conn, 'storage.retrieve',
-        [json.dumps({ "pubkey": my_ss_id }).encode()]
-        )[0])
+    r = omq.request_future(conn, 'storage.retrieve',
+        [json.dumps({
+            "pubkey": my_ss_id,
+            "pubkey_ed25519": sk.verify_key.encode().hex(),
+            "timestamp": ts,
+            "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode(),
+            }).encode()]
+        ).get()
+    assert len(r) == 1
+    r = json.loads(r[0])
     assert not r['messages']
 
 
@@ -98,7 +119,7 @@ def test_non_session_no_ed25519(omq, random_sn, sk, exclude):
 
     swarm = ss.get_swarm(omq, random_sn, xsk, netid=4)
     sn = ss.random_swarm_members(swarm, 1, exclude)[0]
-    conn = omq.connect_remote("curve://{}:{}/{}".format(sn['ip'], sn['port_omq'], sn['pubkey_x25519']))
+    conn = omq.connect_remote(sn_address(sn))
 
     msgs = ss.store_n(omq, conn, xsk, b"omg123", 4)
 
@@ -114,6 +135,6 @@ def test_non_session_no_ed25519(omq, random_sn, sk, exclude):
             "signature": sig
     }
 
-    resp = omq.request(conn, 'storage.delete_all', [json.dumps(params).encode()])
+    resp = omq.request_future(conn, 'storage.delete_all', [json.dumps(params).encode()]).get()
 
     assert resp == [b'400', b'invalid request: pubkey_ed25519 is only permitted for 05[...] pubkeys']

network-tests/test_store_retrieve.py

@@ -1,4 +1,4 @@
-import pyoxenmq
+from util import sn_address
 import ss
 import time
 import base64
@@ -11,17 +11,19 @@ def test_store(omq, random_sn, sk, exclude):
     swarm = ss.get_swarm(omq, random_sn, sk)
 
     sn = ss.random_swarm_members(swarm, 1, exclude)[0]
-    conn = omq.connect_remote("curve://{}:{}/{}".format(sn['ip'], sn['port_omq'], sn['pubkey_x25519']))
+    conn = omq.connect_remote(sn_address(sn))
 
     ts = int(time.time() * 1000)
     ttl = 86400000
     exp = ts + ttl
     # Store a message for myself
-    s = json.loads(omq.request(conn, 'storage.store', [json.dumps({
+    s = omq.request_future(conn, 'storage.store', [json.dumps({
         "pubkey": '05' + sk.verify_key.encode().hex(),
         "timestamp": ts,
         "ttl": ttl,
-        "data": base64.b64encode("abc 123".encode()).decode()}).encode()])[0])
+        "data": base64.b64encode("abc 123".encode()).decode()}).encode()]).get()
+    assert len(s) == 1
+    s = json.loads(s[0])
 
     hash = blake2b("{}{}".format(ts, exp).encode() + b'\x05' + sk.verify_key.encode() + b'abc 123',
             encoder=Base64Encoder).decode().rstrip('=')
@@ -40,36 +42,32 @@ def test_store(omq, random_sn, sk, exclude):
 
 
 def test_store_retrieve_unauthenticated(omq, random_sn, sk, exclude):
-    """Retrieves messages without authentication.  This test will break in the future when we turn
-    on required retrieval signatures"""
+    """Attempts to retrieve messages without authentication.  This should fail (as of HF19)."""
     sns = ss.random_swarm_members(ss.get_swarm(omq, random_sn, sk), 2, exclude)
-    conn1 = omq.connect_remote("curve://{}:{}/{}".format(sns[0]['ip'], sns[0]['port_omq'], sns[0]['pubkey_x25519']))
+    conn1 = omq.connect_remote(sn_address(sns[0]))
 
     ts = int(time.time() * 1000)
     ttl = 86400000
     exp = ts + ttl
     # Store a message for myself
-    s = json.loads(omq.request(conn1, 'storage.store', [json.dumps({
+    s = omq.request_future(conn1, 'storage.store', [json.dumps({
         "pubkey": '05' + sk.verify_key.encode().hex(),
         "timestamp": ts,
         "ttl": ttl,
-        "data": base64.b64encode(b"abc 123").decode()}).encode()])[0])
+        "data": base64.b64encode(b"abc 123").decode()}).encode()]).get()
+    assert len(s) == 1
+    s = json.loads(s[0])
 
     hash = blake2b("{}{}".format(ts, exp).encode() + b'\x05' + sk.verify_key.encode() + b'abc 123',
             encoder=Base64Encoder).decode().rstrip('=')
 
     assert all(v['hash'] == hash for v in s['swarm'].values())
 
-    conn2 = omq.connect_remote("curve://{}:{}/{}".format(sns[1]['ip'], sns[1]['port_omq'], sns[1]['pubkey_x25519']))
-    r = json.loads(omq.request(conn2, 'storage.retrieve', [json.dumps({
-        "pubkey": '05' + sk.verify_key.encode().hex() }).encode()])[0])
+    conn2 = omq.connect_remote(sn_address(sns[1]))
+    r = omq.request_future(conn2, 'storage.retrieve', [json.dumps({
+        "pubkey": '05' + sk.verify_key.encode().hex() }).encode()]).get()
 
-    assert len(r['messages']) == 1
-    msg = r['messages'][0]
-    assert msg['data'] == base64.b64encode(b'abc 123').decode()
-    assert msg['timestamp'] == ts
-    assert msg['expiration'] == exp
-    assert msg['hash'] == hash
+    assert r == [b'401', b'retrieve: request signature required']
 
 
 def test_store_retrieve_authenticated(omq, random_sn, sk, exclude):
@@ -77,30 +75,37 @@ def test_store_retrieve_authenticated(omq, random_sn, sk, exclude):
     xpk = xsk.public_key
     sn_x = ss.random_swarm_members(ss.get_swarm(omq, random_sn, xsk), 1, exclude)[0]
     sn_ed = ss.random_swarm_members(ss.get_swarm(omq, random_sn, sk), 1, exclude)[0]
-    conn_x = omq.connect_remote("curve://{}:{}/{}".format(sn_x['ip'], sn_x['port_omq'], sn_x['pubkey_x25519']))
-    conn_ed = omq.connect_remote("curve://{}:{}/{}".format(sn_ed['ip'], sn_ed['port_omq'], sn_ed['pubkey_x25519']))
+    conn_x = omq.connect_remote(sn_address(sn_x))
+    conn_ed = omq.connect_remote(sn_address(sn_ed))
 
     ts = int(time.time() * 1000)
     ttl = 86400000
     exp = ts + ttl
     # Store message for myself, using both my ed25519 key and x25519 key to test different auth
     # modes
-    s1 = json.loads(omq.request(conn_x, 'storage.store', [json.dumps({
+    s1 = omq.request_future(conn_x, 'storage.store', [json.dumps({
         "pubkey": '05' + xpk.encode().hex(),
         "timestamp": ts,
         "ttl": ttl,
-        "data": base64.b64encode(b"abc 123").decode()}).encode()])[0])
+        "data": base64.b64encode(b"abc 123").decode()}).encode()])
+    s2 = omq.request_future(conn_ed, 'storage.store', [json.dumps({
+        "pubkey": '03' + sk.verify_key.encode().hex(),
+        "timestamp": ts,
+        "ttl": ttl,
+        "data": base64.b64encode(b"def 456").decode()}).encode()])
+
+    s1 = s1.get()
+    assert len(s1) == 1
+    s1 = json.loads(s1[0])
 
     hash1 = blake2b("{}{}".format(ts, exp).encode() + b'\x05' + xpk.encode() + b'abc 123',
             encoder=Base64Encoder).decode().rstrip('=')
 
     assert all(v['hash'] == hash1 for v in s1['swarm'].values())
 
-    s2 = json.loads(omq.request(conn_ed, 'storage.store', [json.dumps({
-        "pubkey": '03' + sk.verify_key.encode().hex(),
-        "timestamp": ts,
-        "ttl": ttl,
-        "data": base64.b64encode(b"def 456").decode()}).encode()])[0])
+    s2 = s2.get()
+    assert len(s2) == 1
+    s2 = json.loads(s2[0])
 
     hash2 = blake2b("{}{}".format(ts, exp).encode() + b'\x03' + sk.verify_key.encode() + b'def 456',
             encoder=Base64Encoder).decode().rstrip('=')
@@ -109,39 +114,40 @@ def test_store_retrieve_authenticated(omq, random_sn, sk, exclude):
     sig = sk.sign(to_sign, encoder=Base64Encoder).signature.decode()
     badsig = sig[0:4] + ('z' if sig[4] != 'z' else 'a') + sig[5:]
 
-    r_good1 = json.loads(omq.request(conn_x, 'storage.retrieve', [
+    r_good1 = omq.request_future(conn_x, 'storage.retrieve', [
         json.dumps({
             "pubkey": '05' + xpk.encode().hex(),
             "timestamp": ts,
             "signature": sig,
             "pubkey_ed25519": sk.verify_key.encode().hex()
-        }).encode()])[0])
-    r_good2 = json.loads(omq.request(conn_ed, 'storage.retrieve', [
+        }).encode()])
+    r_good2 = omq.request_future(conn_ed, 'storage.retrieve', [
         json.dumps({
             "pubkey": '03' + sk.verify_key.encode().hex(),
             "timestamp": ts,
             "signature": sig
-        }).encode()])[0])
-    r_bad1 = omq.request(conn_x, 'storage.retrieve', [
+        }).encode()])
+    r_bad1 = omq.request_future(conn_x, 'storage.retrieve', [
         json.dumps({
             "pubkey": '05' + xpk.encode().hex(),
             "timestamp": ts,
             "signature": badsig,  # invalid sig
             "pubkey_ed25519": sk.verify_key.encode().hex()
         }).encode()])
-    r_bad2 = omq.request(conn_ed, 'storage.retrieve', [
+    r_bad2 = omq.request_future(conn_ed, 'storage.retrieve', [
         json.dumps({
             "pubkey": '03' + sk.verify_key.encode().hex(),
             "timestamp": ts,
             "signature": badsig  # invalid sig
         }).encode()])
-    r_bad3 = omq.request(conn_ed, 'storage.retrieve', [
+    r_bad3 = omq.request_future(conn_ed, 'storage.retrieve', [
         json.dumps({
             "pubkey": '03' + sk.verify_key.encode().hex(),
             "timestamp": ts,
             #"signature": badsig  # has timestamp but missing sig
         }).encode()])
 
+    r_good1 = json.loads(r_good1.get()[0])
     assert len(r_good1['messages']) == 1
     msg = r_good1['messages'][0]
     assert msg['data'] == base64.b64encode(b'abc 123').decode()
@@ -149,6 +155,7 @@ def test_store_retrieve_authenticated(omq, random_sn, sk, exclude):
     assert msg['expiration'] == exp
     assert msg['hash'] == hash1
 
+    r_good2 = json.loads(r_good2.get()[0])
     assert len(r_good2['messages']) == 1
     msg = r_good2['messages'][0]
     assert msg['data'] == base64.b64encode(b'def 456').decode()
@@ -156,9 +163,9 @@ def test_store_retrieve_authenticated(omq, random_sn, sk, exclude):
     assert msg['expiration'] == exp
     assert msg['hash'] == hash2
 
-    assert r_bad1 == [b'401', b'retrieve signature verification failed']
-    assert r_bad2 == [b'401', b'retrieve signature verification failed']
-    assert r_bad3 == [b'400', b"invalid request: Required field 'signature' missing"]
+    assert r_bad1.get() == [b'401', b'retrieve signature verification failed']
+    assert r_bad2.get() == [b'401', b'retrieve signature verification failed']
+    assert r_bad3.get() == [b'400', b"invalid request: Required field 'signature' missing"]
 
 
 def exactly_one(iterable):
@@ -169,7 +176,7 @@ def exactly_one(iterable):
 
 def test_store_retrieve_multiple(omq, random_sn, sk, exclude):
     sns = ss.random_swarm_members(ss.get_swarm(omq, random_sn, sk), 2, exclude)
-    conn1 = omq.connect_remote("curve://{}:{}/{}".format(sns[0]['ip'], sns[0]['port_omq'], sns[0]['pubkey_x25519']))
+    conn1 = omq.connect_remote(sn_address(sns[0]))
 
 
     basemsg = b"This is my message \x00<--that's a null, this is invalid utf8: \x80\xff"
@@ -178,9 +185,13 @@ def test_store_retrieve_multiple(omq, random_sn, sk, exclude):
     msgs = ss.store_n(omq, conn1, sk, basemsg, 5)
 
     # Retrieve all messages from the swarm (should give back the 5 we just stored):
-    conn2 = omq.connect_remote("curve://{}:{}/{}".format(sns[1]['ip'], sns[1]['port_omq'], sns[1]['pubkey_x25519']))
-    resp = omq.request(conn2, 'storage.retrieve', [json.dumps({
-        "pubkey": '05' + sk.verify_key.encode().hex() }).encode()])
+    conn2 = omq.connect_remote(sn_address(sns[1]))
+    ts = int(time.time() * 1000)
+    resp = omq.request_future(conn2, 'storage.retrieve', [json.dumps({
+        "pubkey": '05' + sk.verify_key.encode().hex(),
+        "timestamp": ts,
+        "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode(),
+    }).encode()]).get()
 
     assert len(resp) == 1
     r = json.loads(resp[0])
@@ -198,10 +209,12 @@ def test_store_retrieve_multiple(omq, random_sn, sk, exclude):
     new_msgs = ss.store_n(omq, conn2, sk, basemsg, 6, 1)
 
     # Retrieve using a last_hash so that we should get back only the 6:
-    resp = omq.request(conn1, 'storage.retrieve', [json.dumps({
+    resp = omq.request_future(conn1, 'storage.retrieve', [json.dumps({
         "pubkey": '05' + sk.verify_key.encode().hex(),
-        "last_hash": msgs[4]['hash']
-        }).encode()])
+        "last_hash": msgs[4]['hash'],
+        "timestamp": ts,
+        "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode(),
+        }).encode()]).get()
 
     assert len(resp) == 1
     r = json.loads(resp[0])
@@ -215,8 +228,12 @@ def test_store_retrieve_multiple(omq, random_sn, sk, exclude):
         assert source['req']['expiry'] == m['expiration']
 
     # Give an unknown hash which should retrieve all:
-    r = json.loads(omq.request(conn2, 'storage.retrieve', [json.dumps({
+    r = omq.request_future(conn2, 'storage.retrieve', [json.dumps({
         "pubkey": '05' + sk.verify_key.encode().hex(),
-        "last_hash": "abcdef"
-        }).encode()])[0])
-
+        "last_hash": "0123456789012345678901234567890123456789123",
+        "timestamp": ts,
+        "signature": sk.sign(f"retrieve{ts}".encode(), encoder=Base64Encoder).signature.decode(),
+        }).encode()]).get()
+    assert len(r) == 1
+    r = json.loads(r[0])
+    assert len(r['messages']) == 11

network-tests/util.py

@@ -0,0 +1,4 @@
+from oxenmq import Address
+
+def sn_address(sn):
+    return Address(sn['ip'], sn['port_omq'], bytes.fromhex(sn['pubkey_x25519']))

storage/include/Database.hpp

@@ -21,9 +21,9 @@ class Database {
 
   public:
     // Recommended period for calling clean_expired()
-    inline static constexpr auto CLEANUP_PERIOD = 10s;
+    static constexpr auto CLEANUP_PERIOD = 10s;
 
-    inline static constexpr int64_t SIZE_LIMIT = int64_t(3584) * 1024 * 1024;  // 3.5 GB
+    static constexpr int64_t SIZE_LIMIT = int64_t(3584) * 1024 * 1024;  // 3.5 GB
 
     // Constructor.  Note that you *must* also set up a timer that runs periodically (every
     // CLEANUP_PERIOD is recommended) and calls clean_expired().
@@ -32,7 +32,7 @@ class Database {
     ~Database();
 
     // if the database is full then print an error only once ever N errors
-    inline static constexpr int DB_FULL_FREQUENCY = 100;
+    static constexpr int DB_FULL_FREQUENCY = 100;
 
     // Attempts to store a message in the database.  Returns true if inserted, false on failure
     // due to the message already existing, and nullopt if the insertion failed because the
@@ -44,14 +44,15 @@ class Database {
 
     void bulk_store(const std::vector<message>& items);
 
-    // Retrieves messages owned by pubkey received since `last_hash` (which must also be owned
-    // by pubkey).  If last_hash is empty or not found then returns all messages (up to the
-    // limit). Optionally takes a maximum number of messages to return.
+    // Retrieves messages owned by pubkey received since `last_hash` stored in namespace `ns`.  If
+    // last_hash is empty or not found then returns all messages (up to the limit). Optionally takes
+    // a maximum number of messages to return.
     //
     // Note that the `pubkey` value of the returned message's will be left default constructed,
     // i.e. *not* filled with the given pubkey.
     std::vector<message> retrieve(
             const user_pubkey_t& pubkey,
+            namespace_id ns,
             const std::string& last_hash,
             std::optional<int> num_results = std::nullopt);
 
@@ -71,44 +72,60 @@ class Database {
     std::optional<message> retrieve_random();
 
     // Get message by `msg_hash`, return true if found.  Note that this does *not* filter by
-    // pubkey!
+    // pubkey or namespace!
     std::optional<message> retrieve_by_hash(const std::string& msg_hash);
 
     // Removes expired messages from the database; the `Database` instance owner should call
     // this periodically.
     void clean_expired();
 
-    // Deletes all messages owned by the given pubkey.  Returns the hashes of any deleted
-    // messages on success (including the case where no messages are deleted), nullopt on query
-    // failure.
-    std::vector<std::string> delete_all(const user_pubkey_t& pubkey);
+    // Deletes all messages owned by the given pubkey.  Returns the [namespace, hash] pairs of any
+    // deleted messages.
+    std::vector<std::pair<namespace_id, std::string>> delete_all(const user_pubkey_t& pubkey);
 
-    // Delete a message owned by the given pubkey having the given hashes.  Returns the hashes
-    // of any delete messages on success (including the case where no messages are deleted),
-    // nullopt on query failure.
+    // Deletes all messages owned by the given pubkey with the given namespace.  Returns the hashes
+    // of any deleted messages.
+    std::vector<std::string> delete_all(const user_pubkey_t& pubkey, namespace_id ns);
+
+    // Delete messages owned by the given pubkey having the given hashes.  Returns the hashes of any
+    // deleted messages.
     std::vector<std::string> delete_by_hash(
             const user_pubkey_t& pubkey, const std::vector<std::string>& msg_hashes);
 
-    // Deletes all messages owned by the given pubkey with a timestamp <= timestamp.  Returns
-    // the hashes of any deleted messages (including the case where no messages are deleted),
-    // nullopt on query failure.
-    std::vector<std::string> delete_by_timestamp(
+    // Deletes all messages owned by the given pubkey with a timestamp <= timestamp.  Returns the
+    // [namespace, hash] pairs of any deleted messages.
+    std::vector<std::pair<namespace_id, std::string>> delete_by_timestamp(
             const user_pubkey_t& pubkey, std::chrono::system_clock::time_point timestamp);
 
-    // Shortens the expiry time of the given messages owned by the given pubkey.  Expiries can
-    // only be shortened (i.e. brought closer to now), not extended into the future.  Returns a
-    // vector of hashes of messages that had their expiries updates.  (Missing messages and
-    // messages that already had an expiry <= the given expiry value are not returned).
+    // Deletes all messages owned by the given pubkey with a timestamp <= timestamp in the given
+    // namespace.  Returns the hashes of any deleted messages.
+    std::vector<std::string> delete_by_timestamp(
+            const user_pubkey_t& pubkey,
+            namespace_id ns,
+            std::chrono::system_clock::time_point timestamp);
+
+    // Shortens the expiry time of the given messages owned by the given pubkey.  Expiries can only
+    // be shortened (i.e. brought closer to now), not extended into the future.  Returns a vector of
+    // hashes of messages that had their expiries updates.  (Missing messages and messages that
+    // already had an expiry <= the given expiry value are not returned).
     std::vector<std::string> update_expiry(
             const user_pubkey_t& pubkey,
             const std::vector<std::string>& msg_hashes,
             std::chrono::system_clock::time_point new_exp);
 
     // Shortens the expiry time of all messages owned by the given pubkey.  Expiries can only be
-    // shortened (i.e. brought closer to now), not extended into the future.  Returns a vector
-    // of hashes of messages that had their expiries shorten.
-    std::vector<std::string> update_all_expiries(
+    // shortened (i.e. brought closer to now), not extended into the future.  Returns a vector of
+    // [namespace, hash] pairs of messages that had their expiries shortened.
+    std::vector<std::pair<namespace_id, std::string>> update_all_expiries(
             const user_pubkey_t& pubkey, std::chrono::system_clock::time_point new_exp);
+
+    // Shortens the expiry time of all messages owned by the given pubkey in the given namespace.
+    // Expiries can only be shortened (i.e. brought closer to now), not extended into the future.
+    // Returns a vector of hashes of messages that had their expiries shortened.
+    std::vector<std::string> update_all_expiries(
+            const user_pubkey_t& pubkey,
+            namespace_id ns,
+            std::chrono::system_clock::time_point new_exp);
 };
 
 }  // namespace oxen

storage/src/Database.cpp

@@ -12,7 +12,9 @@
 #include <exception>
 #include <shared_mutex>
 #include <thread>
+#include <type_traits>
 #include <unordered_set>
+#include <utility>
 
 #include <SQLiteCpp/SQLiteCpp.h>
 #include <sqlite3.h>
@@ -58,7 +60,9 @@ namespace {
         else if constexpr (std::is_same_v<T, user_pubkey_t>) {
             bind_blob_ref(st, i++, val.raw());
             st.bind(i++, val.type());
-        } else
+        } else if constexpr (std::is_same_v<T, namespace_id>)
+            st.bind(i++, static_cast<std::underlying_type_t<namespace_id>>(val));
+        else
             st.bind(i++, val);
     }
 
@@ -95,20 +99,63 @@ namespace {
     using first_type_t = typename first_type<T...>::type;
 
     template <typename... T>
-    using type_or_tuple =
-            std::conditional_t<sizeof...(T) == 1, first_type_t<T...>, std::tuple<T...>>;
+    struct tuple_or_pair_impl {
+        using type = std::tuple<T...>;
+    };
+    template <typename T1, typename T2>
+    struct tuple_or_pair_impl<T1, T2> {
+        using type = std::pair<T1, T2>;
+    };
+
+    // Converts a parameter pack T... into either a plain T (if singleton), a pair (if exactly 2),
+    // or a tuple<T...>:
+    template <typename... T>
+    using type_or_tuple = std::conditional_t<
+            sizeof...(T) == 1,
+            first_type_t<T...>,
+            typename tuple_or_pair_impl<T...>::type>;
+
+    // We want to keep namespace_id as a type-safe integer, which requires some working around here
+    // to get an int64_t out of the database and stuff it into a namespace_id; everything else we
+    // pass through untouched.
+    template <typename... T>
+    constexpr bool contains_namespace_id = (... || std::is_same_v<T, namespace_id>);
+
+    template <typename T>
+    using db_source_type = std::conditional_t<std::is_same_v<T, namespace_id>, int64_t, T>;
+
+    template <typename T>
+    std::conditional_t<std::is_same_v<T, namespace_id>, namespace_id, T&> transform_db_source_impl(
+            db_source_type<T>& source_val) {
+        if constexpr (std::is_same_v<T, namespace_id>)
+            return static_cast<namespace_id>(source_val);
+        else
+            return source_val;
+    }
+
+    template <typename... T, size_t... I>
+    type_or_tuple<T...> transform_db_source(
+            std::tuple<db_source_type<T>...>&& source, std::index_sequence<I...>) {
+        return {std::move(transform_db_source_impl<T>(std::get<I>(source)))...};
+    }
 
     // Retrieves a single row of values from the current state of a statement (i.e. after a
-    // executeStep() call that is expecting a return value).  If `T...` is a single type then
-    // this returns the single T value; if T... has multiple types then you get back a tuple of
-    // values.
-    template <typename T>
-    T get(SQLite::Statement& st) {
-        return static_cast<T>(st.getColumn(0));
-    }
-    template <typename T1, typename T2, typename... Tn>
-    std::tuple<T1, T2, Tn...> get(SQLite::Statement& st) {
-        return st.getColumns<std::tuple<T1, T2, Tn...>, 2 + sizeof...(Tn)>();
+    // executeStep() call that is expecting a return value).  If `T...` is a single type then this
+    // returns the single T value; if T... is two values you get back a pair, otherwise you get back
+    // a tuple of values.
+    template <typename... T>
+    type_or_tuple<T...> get(SQLite::Statement& st) {
+        if constexpr (contains_namespace_id<T...>) {
+            return transform_db_source<T...>(
+                    get<std::conditional_t<std::is_same_v<T, namespace_id>, int64_t, T>...>(st),
+                    std::make_index_sequence<sizeof...(T)>{});
+        } else {
+            using TT = type_or_tuple<T...>;
+            if constexpr (sizeof...(T) == 1)
+                return static_cast<TT>(st.getColumn(0));
+            else
+                return st.getColumns<TT, sizeof...(T)>();
+        }
     }
 
     // Steps a statement to completion that is expected to return at most one row, optionally
@@ -148,8 +195,9 @@ namespace {
         return *std::move(maybe_result);
     }
 
-    // Executes a query to completion, collecting each row into a vector<T> (or
-    // vector<tuple<T...>> if multiple T are given).  Can optionally bind before executing.
+    // Executes a query to completion, collecting each row into a vector<T>, vector<pair<T1,T2>, or
+    // vector<tuple<T...>>, depending on whether 1, 2, or more Ts are given.  Can optionally bind
+    // before executing.
     template <typename... T, typename... Bind>
     std::vector<type_or_tuple<T...>> get_all(SQLite::Statement& st, const Bind&... bind) {
         int i = 1;
@@ -218,6 +266,28 @@ class DatabaseImpl {
         if (!db.tableExists("owners")) {
             create_schema();
         }
+
+        bool have_namespace = false;
+        SQLite::Statement msg_cols{db, "PRAGMA main.table_info(messages)"};
+        while (msg_cols.executeStep()) {
+            auto [cid, name] = get<int64_t, std::string>(msg_cols);
+            if (name == "namespace")
+                have_namespace = true;
+        }
+
+        if (!have_namespace) {
+            OXEN_LOG(info, "Upgrading database schema");
+            db.exec(R"(
+DROP INDEX IF EXISTS messages_owner;
+DROP TRIGGER IF EXISTS owned_messages_insert;
+DROP VIEW IF EXISTS owned_messages;
+ALTER TABLE messages ADD COLUMN namespace INTEGER NOT NULL DEFAULT 0;
+            )");
+        }
+
+        views_triggers_indices();
+
+        OXEN_LOG(info, "Database setup complete");
     }
 
     void create_schema() {
@@ -236,39 +306,13 @@ CREATE TABLE messages (
     id INTEGER PRIMARY KEY,
     hash TEXT NOT NULL,
     owner INTEGER NOT NULL REFERENCES owners(id),
+    namespace INTEGER NOT NULL DEFAULT 0,
     timestamp INTEGER NOT NULL,
     expiry INTEGER NOT NULL,
     data BLOB NOT NULL,
 
     UNIQUE(hash)
 );
-
-CREATE INDEX messages_expiry ON messages(expiry);
-CREATE INDEX messages_owner ON messages(owner, timestamp);
-
-CREATE TRIGGER owner_autoclean
-    AFTER DELETE ON messages FOR EACH ROW WHEN NOT EXISTS (SELECT * FROM messages WHERE owner = old.owner)
-    BEGIN
-        DELETE FROM owners WHERE id = old.owner;
-    END;
-
-CREATE VIEW owned_messages AS
-    SELECT owners.id AS oid, type, pubkey, messages.id AS mid, hash, timestamp, expiry, data
-    FROM messages JOIN owners ON messages.owner = owners.id;
-
-CREATE TRIGGER owned_messages_insert
-    INSTEAD OF INSERT ON owned_messages FOR EACH ROW WHEN NEW.oid IS NULL
-    BEGIN
-        INSERT INTO owners (type, pubkey) VALUES (NEW.type, NEW.pubkey) ON CONFLICT DO NOTHING;
-        INSERT INTO messages values (
-            NEW.mid,
-            NEW.hash,
-            (SELECT id FROM owners WHERE type = NEW.type AND pubkey = NEW.pubkey),
-            NEW.timestamp,
-            NEW.expiry,
-            NEW.data);
-    END;
-
         )");
 
         if (db.tableExists("Data")) {
@@ -295,8 +339,8 @@ CREATE TRIGGER owned_messages_insert
                 int type;
                 std::array<char, 32> pubkey;
                 std::string old_owner = old_owners.getColumn(0);
-                if (old_owner.size() == 66 && util::starts_with(old_owner, "05")
-                    && oxenc::is_hex(old_owner)) {
+                if (old_owner.size() == 66 && util::starts_with(old_owner, "05") &&
+                    oxenc::is_hex(old_owner)) {
                     type = 5;
                     oxenc::from_hex(old_owner.begin() + 2, old_owner.end(), pubkey.begin());
                 } else if (old_owner.size() == 64 && oxenc::is_hex(old_owner)) {
@@ -348,8 +392,46 @@ CREATE TRIGGER owned_messages_insert
         }
 
         transaction.commit();
+    }
 
-        OXEN_LOG(info, "Database setup complete");
+    void views_triggers_indices() {
+        // We create these separate from the table because it makes upgrading easier (we can just
+        // drop the indices/views that we want to recreate).
+
+        SQLite::Transaction transaction{db};
+
+        db.exec(R"(
+CREATE TRIGGER IF NOT EXISTS owner_autoclean
+    AFTER DELETE ON messages FOR EACH ROW WHEN NOT EXISTS (SELECT * FROM messages WHERE owner = old.owner)
+    BEGIN
+        DELETE FROM owners WHERE id = old.owner;
+    END;
+
+CREATE INDEX IF NOT EXISTS messages_expiry ON messages(expiry);
+CREATE INDEX IF NOT EXISTS messages_owner ON messages(owner, namespace, timestamp);
+CREATE INDEX IF NOT EXISTS messages_hash ON messages(hash);
+
+CREATE VIEW IF NOT EXISTS owned_messages AS
+    SELECT owners.id AS oid, type, pubkey, messages.id AS mid, hash, namespace, timestamp, expiry, data
+    FROM messages JOIN owners ON messages.owner = owners.id;
+
+CREATE TRIGGER IF NOT EXISTS owned_messages_insert
+    INSTEAD OF INSERT ON owned_messages FOR EACH ROW WHEN NEW.oid IS NULL
+    BEGIN
+        INSERT INTO owners (type, pubkey) VALUES (NEW.type, NEW.pubkey) ON CONFLICT DO NOTHING;
+        INSERT INTO messages (id, hash, owner, namespace, timestamp, expiry, data) VALUES (
+            NEW.mid,
+            NEW.hash,
+            (SELECT id FROM owners WHERE type = NEW.type AND pubkey = NEW.pubkey),
+            NEW.namespace,
+            NEW.timestamp,
+            NEW.expiry,
+            NEW.data
+        );
+    END;
+        )");
+
+        transaction.commit();
     }
 
     /** Wrapper around a SQLite::Statement that calls `tryReset()` on destruction of the
@@ -430,11 +512,13 @@ static std::optional<message> get_message(DatabaseImpl& impl, SQLite::Statement&
     std::optional<message> msg;
     while (st.executeStep()) {
         assert(!msg);
-        auto [hash, otype, opubkey, ts, exp, data] =
-                get<std::string, uint8_t, std::string, int64_t, int64_t, std::string>(st);
+        auto [hash, otype, opubkey, ns, ts, exp, data] =
+                get<std::string, uint8_t, std::string, namespace_id, int64_t, int64_t, std::string>(
+                        st);
         msg.emplace(
                 impl.load_pubkey(otype, std::move(opubkey)),
                 std::move(hash),
+                ns,
                 from_epoch_ms(ts),
                 from_epoch_ms(exp),
                 std::move(data));
@@ -445,7 +529,7 @@ static std::optional<message> get_message(DatabaseImpl& impl, SQLite::Statement&
 std::optional<message> Database::retrieve_random() {
     clean_expired();
     auto st = impl->prepared_st(
-            "SELECT hash, type, pubkey, timestamp, expiry, data"
+            "SELECT hash, type, pubkey, namespace, timestamp, expiry, data"
             " FROM owned_messages "
             " WHERE mid = (SELECT id FROM messages ORDER BY RANDOM() LIMIT 1)");
     return get_message(*impl, st);
@@ -453,7 +537,7 @@ std::optional<message> Database::retrieve_random() {
 
 std::optional<message> Database::retrieve_by_hash(const std::string& msg_hash) {
     auto st = impl->prepared_st(
-            "SELECT hash, type, pubkey, timestamp, expiry, data"
+            "SELECT hash, type, pubkey, namespace, timestamp, expiry, data"
             " FROM owned_messages WHERE hash = ?");
     st->bindNoCopy(1, msg_hash);
     return get_message(*impl, st);
@@ -461,14 +545,15 @@ std::optional<message> Database::retrieve_by_hash(const std::string& msg_hash) {
 
 std::optional<bool> Database::store(const message& msg) {
     auto st = impl->prepared_st(
-            "INSERT INTO owned_messages"
-            " (pubkey, type, hash, timestamp, expiry, data) VALUES (?, ?, ?, ?, ?, ?)");
+            "INSERT INTO owned_messages (pubkey, type, hash, namespace, timestamp, expiry, data)"
+            " VALUES (?, ?, ?, ?, ?, ?, ?)");
 
     try {
         exec_query(
                 st,
                 msg.pubkey,
                 msg.hash,
+                msg.msg_namespace,
                 to_epoch_ms(msg.timestamp),
                 to_epoch_ms(msg.expiry),
                 blob_binder{msg.data});
@@ -513,7 +598,8 @@ void Database::bulk_store(const std::vector<message>& items) {
     }
 
     auto insert_message = impl->prepared_st(
-            "INSERT INTO messages (owner, hash, timestamp, expiry, data) VALUES (?, ?, ?, ?, ?)"
+            "INSERT INTO messages (owner, hash, namespace, timestamp, expiry, data)"
+            " VALUES (?, ?, ?, ?, ?, ?)"
             " ON CONFLICT DO NOTHING");
 
     for (auto& m : items) {
@@ -527,6 +613,7 @@ void Database::bulk_store(const std::vector<message>& items) {
                 insert_message,
                 owner_it->second,
                 m.hash,
+                m.msg_namespace,
                 to_epoch_ms(m.timestamp),
                 to_epoch_ms(m.expiry),
                 blob_binder{m.data});
@@ -537,7 +624,10 @@ void Database::bulk_store(const std::vector<message>& items) {
 }
 
 std::vector<message> Database::retrieve(
-        const user_pubkey_t& pubkey, const std::string& last_hash, std::optional<int> num_results) {
+        const user_pubkey_t& pubkey,
+        namespace_id ns,
+        const std::string& last_hash,
+        std::optional<int> num_results) {
     std::vector<message> results;
 
     auto owner_st = impl->prepared_st("SELECT id FROM owners WHERE pubkey = ? AND type = ?");
@@ -547,14 +637,15 @@ std::vector<message> Database::retrieve(
 
     std::optional<int64_t> last_id;
     if (!last_hash.empty()) {
-        auto st = impl->prepared_st("SELECT id FROM messages WHERE owner = ? AND hash = ?");
-        last_id = exec_and_maybe_get<int64_t>(st, *ownerid, last_hash);
+        auto st = impl->prepared_st(
+                "SELECT id FROM messages WHERE owner = ? AND namespace = ? AND hash = ?");
+        last_id = exec_and_maybe_get<int64_t>(st, *ownerid, to_int(ns), last_hash);
     }
 
     auto st = impl->prepared_st(
-            last_id ? "SELECT hash, timestamp, expiry, data FROM messages "
+            last_id ? "SELECT hash, namespace, timestamp, expiry, data FROM messages "
                       "WHERE owner = ? AND id > ? ORDER BY id LIMIT ?"
-                    : "SELECT hash, timestamp, expiry, data FROM messages "
+                    : "SELECT hash, namespace, timestamp, expiry, data FROM messages "
                       "WHERE owner = ? ORDER BY id LIMIT ?");
     st->bind(1, *ownerid);
     if (last_id)
@@ -562,9 +653,10 @@ std::vector<message> Database::retrieve(
     st->bind(last_id ? 3 : 2, num_results.value_or(-1));
 
     while (st->executeStep()) {
-        auto [hash, ts, exp, data] = get<std::string, int64_t, int64_t, std::string>(st);
+        auto [hash, ns, ts, exp, data] =
+                get<std::string, namespace_id, int64_t, int64_t, std::string>(st);
         results.emplace_back(
-                std::move(hash), from_epoch_ms(ts), from_epoch_ms(exp), std::move(data));
+                std::move(hash), ns, from_epoch_ms(ts), from_epoch_ms(exp), std::move(data));
     }
 
     return results;
@@ -573,15 +665,17 @@ std::vector<message> Database::retrieve(
 std::vector<message> Database::retrieve_all() {
     std::vector<message> results;
     auto st = impl->prepared_st(
-            "SELECT type, pubkey, hash, timestamp, expiry, data"
+            "SELECT type, pubkey, hash, namespace, timestamp, expiry, data"
             " FROM owned_messages ORDER BY mid");
 
     while (st->executeStep()) {
-        auto [type, pubkey, hash, ts, exp, data] =
-                get<uint8_t, std::string, std::string, int64_t, int64_t, std::string>(st);
+        auto [type, pubkey, hash, ns, ts, exp, data] =
+                get<uint8_t, std::string, std::string, namespace_id, int64_t, int64_t, std::string>(
+                        st);
         results.emplace_back(
                 impl->load_pubkey(type, pubkey),
                 std::move(hash),
+                ns,
                 from_epoch_ms(ts),
                 from_epoch_ms(exp),
                 std::move(data));
@@ -590,34 +684,47 @@ std::vector<message> Database::retrieve_all() {
     return results;
 }
 
-std::vector<std::string> Database::delete_all(const user_pubkey_t& pubkey) {
+std::vector<std::pair<namespace_id, std::string>> Database::delete_all(
+        const user_pubkey_t& pubkey) {
     auto st = impl->prepared_st(
-            "DELETE FROM messages WHERE owner = (SELECT id FROM owners WHERE pubkey = ? AND type = "
-            "?)"
-            " RETURNING hash");
-    return get_all<std::string>(st, pubkey);
+            "DELETE FROM messages"
+            " WHERE owner = (SELECT id FROM owners WHERE pubkey = ? AND type = ?)"
+            " RETURNING namespace, hash");
+    return get_all<namespace_id, std::string>(st, pubkey);
 }
 
-static std::string multi_in_query(std::string_view prefix, size_t count, std::string_view suffix) {
-    std::string query;
-    query.reserve(prefix.size() + (count == 0 ? 0 : 2 * count - 1) + suffix.size());
-    query += prefix;
-    for (size_t i = 0; i < count; i++) {
-        if (i > 0)
-            query += ',';
-        query += '?';
-    }
-    query += suffix;
-    return query;
+std::vector<std::string> Database::delete_all(const user_pubkey_t& pubkey, namespace_id ns) {
+    auto st = impl->prepared_st(
+            "DELETE FROM messages"
+            " WHERE owner = (SELECT id FROM owners WHERE pubkey = ? AND type = ?)"
+            " AND namespace = ?"
+            " RETURNING hash");
+    return get_all<std::string>(st, pubkey, ns);
 }
 
+namespace {
+    std::string multi_in_query(std::string_view prefix, size_t count, std::string_view suffix) {
+        std::string query;
+        query.reserve(prefix.size() + (count == 0 ? 0 : 2 * count - 1) + suffix.size());
+        query += prefix;
+        for (size_t i = 0; i < count; i++) {
+            if (i > 0)
+                query += ',';
+            query += '?';
+        }
+        query += suffix;
+        return query;
+    }
+}  // namespace
+
 std::vector<std::string> Database::delete_by_hash(
         const user_pubkey_t& pubkey, const std::vector<std::string>& msg_hashes) {
     if (msg_hashes.size() == 1) {
         // Use an optimized prepared statement for very common single-hash deletions
         auto st = impl->prepared_st(
                 "DELETE FROM messages"
-                " WHERE owner = (SELECT id FROM owners WHERE pubkey = ? AND type = ?) AND hash = ?"
+                " WHERE owner = (SELECT id FROM owners WHERE pubkey = ? AND type = ?)"
+                " AND hash = ?"
                 " RETURNING hash");
         return get_all<std::string>(st, pubkey, msg_hashes[0]);
     }
@@ -625,9 +732,9 @@ std::vector<std::string> Database::delete_by_hash(
     SQLite::Statement st{
             impl->db,
             multi_in_query(
-                    "DELETE FROM messages "
-                    "WHERE owner = (SELECT id FROM owners WHERE pubkey = ? AND type = ?) AND "
-                    "hash IN ("sv,  // ?,?,?,...,?
+                    "DELETE FROM messages"
+                    " WHERE owner = (SELECT id FROM owners WHERE pubkey = ? AND type = ?)"
+                    " AND hash IN ("sv,  // ?,?,?,...,?
                     msg_hashes.size(),
                     ") RETURNING hash"sv)};
 
@@ -637,13 +744,26 @@ std::vector<std::string> Database::delete_by_hash(
     return get_all<std::string>(st);
 }
 
-std::vector<std::string> Database::delete_by_timestamp(
+std::vector<std::pair<namespace_id, std::string>> Database::delete_by_timestamp(
         const user_pubkey_t& pubkey, std::chrono::system_clock::time_point timestamp) {
     auto st = impl->prepared_st(
             "DELETE FROM messages"
             " WHERE owner = (SELECT id FROM owners WHERE pubkey = ? AND type = ?)"
-            " AND timestamp <= ? RETURNING hash");
-    return get_all<std::string>(st, pubkey, to_epoch_ms(timestamp));
+            " AND timestamp <= ?"
+            " RETURNING hash");
+    return get_all<namespace_id, std::string>(st, pubkey, to_epoch_ms(timestamp));
+}
+
+std::vector<std::string> Database::delete_by_timestamp(
+        const user_pubkey_t& pubkey,
+        namespace_id ns,
+        std::chrono::system_clock::time_point timestamp) {
+    auto st = impl->prepared_st(
+            "DELETE FROM messages"
+            " WHERE owner = (SELECT id FROM owners WHERE pubkey = ? AND type = ?)"
+            " AND timestamp <= ? AND namespace = ?"
+            " RETURNING hash");
+    return get_all<std::string>(st, pubkey, to_epoch_ms(timestamp), ns);
 }
 
 std::vector<std::string> Database::update_expiry(
@@ -665,11 +785,10 @@ std::vector<std::string> Database::update_expiry(
     SQLite::Statement st{
             impl->db,
             multi_in_query(
-                    "UPDATE messages SET expiry = ? "
-                    "WHERE expiry > ? AND owner = (SELECT id FROM owners WHERE pubkey = ? AND type "
-                    "= "
-                    "?) "
-                    "AND hash IN ("sv,  // ?,?,?,...,?
+                    "UPDATE messages SET expiry = ?"
+                    " WHERE expiry > ? "
+                    " AND owner = (SELECT id FROM owners WHERE pubkey = ? AND type = ?)"
+                    " AND hash IN ("sv,  // ?,?,?,...,?
                     msg_hashes.size(),
                     ") RETURNING hash"sv)};
     st.bind(1, new_exp_ms);
@@ -681,14 +800,27 @@ std::vector<std::string> Database::update_expiry(
     return get_all<std::string>(st);
 }
 
-std::vector<std::string> Database::update_all_expiries(
+std::vector<std::pair<namespace_id, std::string>> Database::update_all_expiries(
         const user_pubkey_t& pubkey, std::chrono::system_clock::time_point new_exp) {
     auto new_exp_ms = to_epoch_ms(new_exp);
     auto st = impl->prepared_st(
-            "UPDATE messages SET expiry = ? "
-            "WHERE expiry > ? AND owner = (SELECT id FROM owners WHERE pubkey = ? AND type = ?) "
-            "RETURNING hash");
-    return get_all<std::string>(st, new_exp_ms, new_exp_ms, pubkey);
+            "UPDATE messages SET expiry = ?"
+            " WHERE expiry > ? AND owner = (SELECT id FROM owners WHERE pubkey = ? AND type = ?)"
+            " RETURNING namespace, hash");
+    return get_all<namespace_id, std::string>(st, new_exp_ms, new_exp_ms, pubkey);
+}
+
+std::vector<std::string> Database::update_all_expiries(
+        const user_pubkey_t& pubkey,
+        namespace_id ns,
+        std::chrono::system_clock::time_point new_exp) {
+    auto new_exp_ms = to_epoch_ms(new_exp);
+    auto st = impl->prepared_st(
+            "UPDATE messages SET expiry = ?"
+            " WHERE expiry > ? AND owner = (SELECT id FROM owners WHERE pubkey = ? AND type = ?)"
+            " AND namespace = ?"
+            " RETURNING hash");
+    return get_all<std::string>(st, new_exp_ms, new_exp_ms, pubkey, ns);
 }
 
 }  // namespace oxen

unit_test/service_node.cpp

@@ -80,9 +80,9 @@ TEST_CASE("service nodes - message hashing", "[service-nodes][messages]") {
     auto expected = "rY7K5YXNsg7d8LBP6R4OoOr6L7IMFxa3Tr8ca5v5nBI";
     CHECK(computeMessageHash(timestamp, expiry, pk, data) == expected);
     CHECK(oxen::compute_hash_blake2b_b64(
-                  {std::to_string(oxen::to_epoch_ms(timestamp))
-                   + std::to_string(oxen::to_epoch_ms(expiry)) + pk.prefixed_raw() + data})
-          == expected);
+                  {std::to_string(oxen::to_epoch_ms(timestamp)) +
+                   std::to_string(oxen::to_epoch_ms(expiry)) + pk.prefixed_raw() + data}) ==
+          expected);
 }
 
 TEST_CASE("service nodes - pubkey to swarm id") {

unit_test/storage.cpp

@@ -35,10 +35,11 @@ TEST_CASE("storage - data persistence", "[storage]") {
     const auto hash = "myhash";
     const auto bytes = "bytesasstring";
     const auto ttl = 123456ms;
+    const auto ns = namespace_id::Default;
     const auto now = std::chrono::system_clock::now();
     {
         Database storage{"."};
-        CHECK(storage.store({pubkey, hash, now, now + ttl, bytes}));
+        CHECK(storage.store({pubkey, hash, ns, now, now + ttl, bytes}));
 
         CHECK(storage.get_owner_count() == 1);
         CHECK(storage.get_message_count() == 1);
@@ -57,6 +58,44 @@ TEST_CASE("storage - data persistence", "[storage]") {
         REQUIRE(items.size() == 1);
         CHECK_FALSE(items[0].pubkey);  // pubkey is left unset when we retrieve for pubkey
         CHECK(items[0].hash == hash);
+        CHECK(items[0].msg_namespace == namespace_id::Default);
+        CHECK(items[0].expiry - items[0].timestamp == ttl);
+        CHECK(items[0].data == bytes);
+    }
+}
+
+TEST_CASE("storage - data persistence, namespace", "[storage][namespace]") {
+    StorageDeleter fixture;
+
+    user_pubkey_t pubkey;
+    REQUIRE(pubkey.load("050123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"));
+    const auto hash = "myhash";
+    const auto bytes = "bytesasstring";
+    const auto ttl = 123456ms;
+    const namespace_id ns{42};
+    const auto now = std::chrono::system_clock::now();
+    {
+        Database storage{"."};
+        CHECK(storage.store({pubkey, hash, ns, now, now + ttl, bytes}));
+
+        CHECK(storage.get_owner_count() == 1);
+        CHECK(storage.get_message_count() == 1);
+
+        // the database is closed when storage goes out of scope
+    }
+    {
+        // re-open the database
+        Database storage{"."};
+
+        CHECK(storage.get_owner_count() == 1);
+        CHECK(storage.get_message_count() == 1);
+
+        auto items = storage.retrieve(pubkey, "");
+
+        REQUIRE(items.size() == 1);
+        CHECK_FALSE(items[0].pubkey);  // pubkey is left unset when we retrieve for pubkey
+        CHECK(items[0].hash == hash);
+        CHECK(items[0].msg_namespace == namespace_id{42});
         CHECK(items[0].expiry - items[0].timestamp == ttl);
         CHECK(items[0].data == bytes);
     }

utils/include/string_utils.hpp

@@ -18,8 +18,8 @@ using namespace std::literals;
 /// Returns true if the first string is equal to the second string, compared case-insensitively.
 inline bool string_iequal(std::string_view s1, std::string_view s2) {
     return std::equal(s1.begin(), s1.end(), s2.begin(), s2.end(), [](char a, char b) {
-        return std::tolower(static_cast<unsigned char>(a))
-            == std::tolower(static_cast<unsigned char>(b));
+        return std::tolower(static_cast<unsigned char>(a)) ==
+               std::tolower(static_cast<unsigned char>(b));
     });
 }