oxen-io/oxen-storage-server
1
0
Fork 0
mirror of https://github.com/oxen-io/oxen-storage-server.git synced 2023-12-13 21:00:26 +01:00
Code Issues Projects Releases Wiki Activity

Limit client retrieve request to 10 messages at a time

Browse source
This commit is contained in:
sachaaaaa 2019-05-14 15:59:24 +10:00
parent ebefe5ca02
commit f121c735ed
5 changed files with 63 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
httpserver/service_node.cpp
View file

@ -84,6 +84,7 @@ constexpr std::chrono::milliseconds SWARM_UPDATE_INTERVAL = 200ms;
#else
constexpr std::chrono::milliseconds SWARM_UPDATE_INTERVAL = 1000ms;
#endif
constexpr int CLIENT_RETRIEVE_MESSAGE_LIMIT = 10;
static std::shared_ptr<request_t> make_post_request(const char* target,
std::string&& data) {
@ -632,7 +633,7 @@ void ServiceNode::initiate_peer_test() {
void ServiceNode::bootstrap_peers(const std::vector<sn_record_t>& peers) const {
std::vector<Item> all_entries;
db_->retrieve("", all_entries, "");
get_all_messages(all_entries);
relay_messages(all_entries, peers);
}
@ -670,7 +671,7 @@ void ServiceNode::bootstrap_swarms(
const auto& all_swarms = swarm_->all_swarms();
std::vector<Item> all_entries;
if (!db_->retrieve("", all_entries, "")) {
if (!get_all_messages(all_entries)) {
BOOST_LOG_TRIVIAL(error)
<< "could not retrieve entries from the database\n";
return;
@ -767,10 +768,10 @@ void ServiceNode::salvage_data() const {
bool ServiceNode::retrieve(const std::string& pubKey,
const std::string& last_hash,
std::vector<Item>& items) {
return db_->retrieve(pubKey, items, last_hash);
return db_->retrieve(pubKey, items, last_hash, CLIENT_RETRIEVE_MESSAGE_LIMIT);
}
bool ServiceNode::get_all_messages(std::vector<Item>& all_entries) {
bool ServiceNode::get_all_messages(std::vector<Item>& all_entries) const {
BOOST_LOG_TRIVIAL(trace) << "get all messages";

2
httpserver/service_node.h
View file

@ -190,7 +190,7 @@ class ServiceNode {
/// return all messages for a particular PK (in JSON)
bool
get_all_messages(std::vector<service_node::storage::Item>& all_entries);
get_all_messages(std::vector<service_node::storage::Item>& all_entries) const;
bool retrieve(const std::string& pubKey, const std::string& last_hash,
std::vector<service_node::storage::Item>& items);

2
storage/include/Database.hpp
View file

@ -30,7 +30,7 @@ class Database {
bool retrieve(const std::string& key,
std::vector<service_node::storage::Item>& items,
const std::string& lastHash);
const std::string& lastHash, int num_results = -1);
// Return the total number of messages stored
bool get_message_count(uint64_t& count);

8
storage/src/Database.cpp
View file

@ -119,7 +119,7 @@ void Database::open_and_prepare(const std::string& db_path) {
throw std::runtime_error("could not prepare the bulk save statement");
get_all_for_pk_stmt = prepare_statement(
"SELECT * FROM Data WHERE `Owner` = ? ORDER BY rowid;");
"SELECT * FROM Data WHERE `Owner` = ? ORDER BY rowid LIMIT ?;");
if (!get_all_for_pk_stmt)
throw std::runtime_error(
"could not prepare the get all for pk statement");
@ -131,7 +131,7 @@ void Database::open_and_prepare(const std::string& db_path) {
get_stmt =
prepare_statement("SELECT * FROM `Data` WHERE `Owner` == ? AND rowid >"
"COALESCE((SELECT `rowid` FROM `Data` WHERE `Hash` = "
"?), 0) ORDER BY rowid;");
"?), 0) ORDER BY rowid LIMIT ?;");
if (!get_stmt)
throw std::runtime_error("could not prepare get statement");
@ -338,7 +338,7 @@ bool Database::bulk_store(
}
bool Database::retrieve(const std::string& pubKey, std::vector<Item>& items,
const std::string& lastHash) {
const std::string& lastHash, int num_results) {
sqlite3_stmt* stmt;
@ -347,10 +347,12 @@ bool Database::retrieve(const std::string& pubKey, std::vector<Item>& items,
} else if (lastHash.empty()) {
stmt = get_all_for_pk_stmt;
sqlite3_bind_text(stmt, 1, pubKey.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_int(stmt, 2, num_results);
} else {
stmt = get_stmt;
sqlite3_bind_text(stmt, 1, pubKey.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 2, lastHash.c_str(), -1, SQLITE_STATIC);
sqlite3_bind_int(stmt, 3, num_results);
}
bool success = false;

51
unit_test/storage.cpp
View file

@ -326,4 +326,55 @@ BOOST_AUTO_TEST_CASE(bulk_performance_check) {
<< " ms" << std::endl;
}
}
BOOST_AUTO_TEST_CASE(it_checks_the_retrieve_limit_works) {
StorageRAIIFixture fixture;
boost::asio::io_context ioc;
Database storage(ioc, ".");
const size_t num_entries = 100;
for (size_t i = 0; i < num_entries; i++) {
const auto hash = std::string("hash") + std::to_string(i);
storage.store(hash, "mypubkey", "bytesasstring", 100000,
util::get_time_ms(), "nonce");
}
// should return every items
{
std::vector<service_node::storage::Item> items;
const auto lastHash = "";
BOOST_CHECK(storage.retrieve("mypubkey", items, lastHash));
BOOST_CHECK_EQUAL(items.size(), num_entries);
}
// should return 10 items
{
const int num_results = 10;
std::vector<service_node::storage::Item> items;
const auto lastHash = "";
BOOST_CHECK(storage.retrieve("mypubkey", items, lastHash, num_results));
BOOST_CHECK_EQUAL(items.size(), num_results);
}
// should return 88 items
{
const int num_results = 88;
std::vector<service_node::storage::Item> items;
const auto lastHash = "";
BOOST_CHECK(storage.retrieve("mypubkey", items, lastHash, num_results));
BOOST_CHECK_EQUAL(items.size(), num_results);
}
// should return num_entries items
{
const int num_results = 2 * num_entries;
std::vector<service_node::storage::Item> items;
const auto lastHash = "";
BOOST_CHECK(storage.retrieve("mypubkey", items, lastHash, num_results));
BOOST_CHECK_EQUAL(items.size(), num_entries);
}
}
BOOST_AUTO_TEST_SUITE_END()