oxen-io/oxen-storage-server
1
0
Fork 0
mirror of https://github.com/oxen-io/oxen-storage-server.git synced 2023-12-13 21:00:26 +01:00
Code Issues Projects Releases Wiki Activity

Difficulty history (#155)

Browse source 
* Initial refactor of checkPoW to take a history of difficulties

* Load multiple difficulty levels etc

* Review

* Review changes. Refactor checkPoW to have separate get_valid_difficulty. More sensible names

* Return vector and append payload
This commit is contained in:
Beaudan Campbell-Brown 2019-06-14 16:03:34 +10:00 committed by Maxim Shishmarev
parent 2aa8f15dd5
commit c3a0eadff1
7 changed files with 163 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
httpserver/http_connection.cpp
View file

@ -2,7 +2,6 @@
#include "Database.hpp"
#include "Item.hpp"
#include "channel_encryption.hpp"
#include "pow.hpp"
#include "rate_limiter.h"
#include "serialization.h"
#include "server_certificates.h"
@ -749,16 +748,16 @@ void connection_t::process_store(const json& params) {
// Do not store message if the PoW provided is invalid
std::string messageHash;
const bool validPoW =
const bool valid_pow =
checkPoW(nonce, timestamp, ttl, pubKey, data, messageHash,
service_node_.get_pow_difficulty());
service_node_.get_curr_pow_difficulty());
#ifndef DISABLE_POW
if (!validPoW) {
if (!valid_pow) {
response_.result(432);
response_.set(http::field::content_type, "application/json");
json res_body;
res_body["difficulty"] = service_node_.get_pow_difficulty();
res_body["difficulty"] = service_node_.get_curr_pow_difficulty();
BOOST_LOG_TRIVIAL(error) << "Forbidden. Invalid PoW nonce " << nonce;
/// This might throw if not utf-8 endoded
@ -794,7 +793,7 @@ void connection_t::process_store(const json& params) {
response_.result(http::status::ok);
response_.set(http::field::content_type, "application/json");
json res_body;
res_body["difficulty"] = service_node_.get_pow_difficulty();
res_body["difficulty"] = service_node_.get_curr_pow_difficulty();
body_stream_ << res_body.dump();
BOOST_LOG_TRIVIAL(trace)
<< "Successfully stored message for " << obfuscate_pubkey(pubKey);

62
httpserver/service_node.cpp
View file

@ -5,7 +5,6 @@
#include "http_connection.h"
#include "https_client.h"
#include "lokid_key.h"
#include "pow.hpp"
#include "serialization.h"
#include "signature.h"
#include "utils.hpp"
@ -39,7 +38,8 @@ static void make_sn_request(boost::asio::io_context& ioc,
return make_https_request(ioc, sn_address, port, req, std::move(cb));
}
int query_pow_difficulty() {
std::vector<pow_difficulty_t> query_pow_difficulty(std::error_code& ec) {
std::vector<pow_difficulty_t> new_history;
int response;
unsigned char query_buffer[1024] = {};
response = res_query(POW_DIFFICULTY_URL, ns_c_in, ns_t_txt, query_buffer,
@ -48,25 +48,29 @@ int query_pow_difficulty() {
ns_msg nsMsg;
if (ns_initparse(query_buffer, response, &nsMsg) == -1) {
BOOST_LOG_TRIVIAL(error) << "Failed to retrieve PoW difficulty";
return -1;
ec = std::make_error_code(std::errc::bad_message);
return new_history;
}
ns_rr rr;
if (ns_parserr(&nsMsg, ns_s_an, 0, &rr) == -1) {
BOOST_LOG_TRIVIAL(error) << "Failed to retrieve PoW difficulty";
return -1;
ec = std::make_error_code(std::errc::bad_message);
return new_history;
}
try {
const json difficulty_json =
json::parse(ns_rr_rdata(rr) + 1, nullptr, true);
pow_difficulty =
std::stoi(difficulty_json.at("difficulty").get<std::string>());
BOOST_LOG_TRIVIAL(info)
<< "Read PoW difficulty: " << std::to_string(pow_difficulty);
return pow_difficulty;
const json history = json::parse(ns_rr_rdata(rr) + 1, nullptr, true);
new_history.reserve(history.size());
for (const auto& el : history.items()) {
const std::chrono::milliseconds timestamp(std::stoi(el.key()));
const int difficulty = el.value().get<int>();
new_history.push_back(pow_difficulty_t{timestamp, difficulty});
}
return new_history;
} catch (...) {
BOOST_LOG_TRIVIAL(error) << "Failed to retrieve PoW difficulty";
return -1;
ec = std::make_error_code(std::errc::bad_message);
return new_history;
}
}
@ -148,7 +152,8 @@ static std::shared_ptr<request_t> make_push_request(std::string&& data) {
return make_post_request("/v1/swarms/push", std::move(data));
}
static bool verify_message(const message_t& msg, int pow_difficulty,
static bool verify_message(const message_t& msg,
const std::vector<pow_difficulty_t> history,
const char** error_message = nullptr) {
if (!util::validateTTL(msg.ttl)) {
if (error_message)
@ -162,9 +167,11 @@ static bool verify_message(const message_t& msg, int pow_difficulty,
}
std::string hash;
#ifndef DISABLE_POW
const int difficulty =
get_valid_difficulty(std::to_string(msg.timestamp), history);
if (!checkPoW(msg.nonce, std::to_string(msg.timestamp),
std::to_string(msg.ttl), msg.pub_key, msg.data, hash,
pow_difficulty)) {
difficulty)) {
if (error_message)
*error_message = "Provided PoW nonce is not valid";
return false;
@ -189,9 +196,6 @@ ServiceNode::ServiceNode(boost::asio::io_context& ioc,
pow_update_timer_(worker_ioc), lokid_key_pair_(lokid_key_pair),
lokid_rpc_port_(lokid_rpc_port) {
// Default value
pow_difficulty_.store(100);
char buf[64] = {0};
if (char const* dest =
util::base32z_encode(lokid_key_pair_.public_key, buf)) {
@ -209,7 +213,10 @@ ServiceNode::ServiceNode(boost::asio::io_context& ioc,
lokid_ping_timer_tick();
worker_thread_ = boost::thread([this]() { worker_ioc_.run(); });
boost::asio::post(worker_ioc_, [this]() { pow_difficulty_timer_tick(); });
boost::asio::post(worker_ioc_, [this]() {
pow_difficulty_timer_tick(std::bind(
&ServiceNode::set_difficulty_history, this, std::placeholders::_1));
});
}
ServiceNode::~ServiceNode() {
@ -342,7 +349,7 @@ bool ServiceNode::process_store(const message_t& msg) {
void ServiceNode::process_push(const message_t& msg) {
#ifndef DISABLE_POW
const char* error_msg;
if (!verify_message(msg, pow_difficulty_.load(), &error_msg))
if (!verify_message(msg, pow_history_, &error_msg))
throw std::runtime_error(error_msg);
#endif
save_if_new(msg);
@ -424,14 +431,15 @@ void ServiceNode::on_swarm_update(const block_update_t& bu) {
initiate_peer_test();
}
void ServiceNode::pow_difficulty_timer_tick() {
const int new_difficulty = query_pow_difficulty();
if (new_difficulty != -1) {
pow_difficulty_.store(new_difficulty);
void ServiceNode::pow_difficulty_timer_tick(const pow_dns_callback_t cb) {
std::error_code ec;
std::vector<pow_difficulty_t> new_history = query_pow_difficulty(ec);
if (!ec) {
boost::asio::post(ioc_, std::bind(cb, new_history));
}
pow_update_timer_.expires_after(POW_DIFFICULTY_UPDATE_INTERVAL);
pow_update_timer_.async_wait(
boost::bind(&ServiceNode::pow_difficulty_timer_tick, this));
boost::bind(&ServiceNode::pow_difficulty_timer_tick, this, cb));
}
void ServiceNode::swarm_timer_tick() {
@ -1003,7 +1011,9 @@ bool ServiceNode::retrieve(const std::string& pubKey,
CLIENT_RETRIEVE_MESSAGE_LIMIT);
}
int ServiceNode::get_pow_difficulty() const { return pow_difficulty_.load(); }
int ServiceNode::get_curr_pow_difficulty() const {
return curr_pow_difficulty_.difficulty;
}
bool ServiceNode::get_all_messages(std::vector<Item>& all_entries) const {
@ -1028,7 +1038,7 @@ void ServiceNode::process_push_batch(const std::string& blob) {
#ifndef DISABLE_POW
const auto it = std::remove_if(
messages.begin(), messages.end(), [this](const message_t& message) {
return verify_message(message, pow_difficulty_.load()) == false;
return verify_message(message, pow_history_) == false;
});
messages.erase(it, messages.end());
if (it != messages.end()) {

25
httpserver/service_node.h
View file

@ -1,6 +1,7 @@
#pragma once
#include <Database.hpp>
#include <chrono>
#include <fstream>
#include <iostream>
#include <memory>
@ -14,6 +15,7 @@
#include "common.h"
#include "lokid_key.h"
#include "pow.hpp"
#include "swarm.h"
static constexpr size_t BLOCK_HASH_CACHE_SIZE = 10;
@ -54,7 +56,10 @@ struct snode_stats_t {
uint64_t relay_fails = 0;
};
int query_pow_difficulty();
using pow_dns_callback_t =
std::function<void(const std::vector<pow_difficulty_t>&)>;
std::vector<pow_difficulty_t> query_pow_difficulty(std::error_code& ec);
/// Represents failed attempt at communicating with a SNode
/// (currently only for single messages)
@ -91,7 +96,9 @@ class ServiceNode {
boost::asio::io_context& worker_ioc_;
boost::thread worker_thread_;
std::atomic<int> pow_difficulty_;
pow_difficulty_t curr_pow_difficulty_{std::chrono::milliseconds(0), 100};
std::vector<pow_difficulty_t> pow_history_{curr_pow_difficulty_};
uint64_t block_height_ = 0;
const uint16_t lokid_rpc_port_;
std::string block_hash_ = "";
@ -151,7 +158,7 @@ class ServiceNode {
void swarm_timer_tick();
/// Update PoW difficulty from DNS text record
void pow_difficulty_timer_tick();
void pow_difficulty_timer_tick(const pow_dns_callback_t cb);
/// Ping the storage server periodically as required for uptime proofs
void lokid_ping_timer_tick();
@ -230,10 +237,20 @@ class ServiceNode {
std::vector<service_node::storage::Item>& all_entries) const;
// Return the current PoW difficulty
int get_pow_difficulty() const;
int get_curr_pow_difficulty() const;
bool retrieve(const std::string& pubKey, const std::string& last_hash,
std::vector<service_node::storage::Item>& items);
void
set_difficulty_history(const std::vector<pow_difficulty_t>& new_history) {
pow_history_ = new_history;
for (const auto& difficulty : pow_history_) {
if (curr_pow_difficulty_.timestamp < difficulty.timestamp) {
curr_pow_difficulty_ = difficulty;
}
}
}
};
} // namespace loki

2
pow/CMakeLists.txt
View file

@ -7,7 +7,7 @@ add_library(pow STATIC ${SOURCES})
loki_add_subdirectory(../utils utils)
set_property(TARGET pow PROPERTY CXX_STANDARD 11)
set_property(TARGET pow PROPERTY CXX_STANDARD 14)
find_package(OpenSSL REQUIRED)
target_link_libraries(pow PRIVATE OpenSSL::SSL)

11
pow/include/pow.hpp
View file

@ -1,7 +1,16 @@
#include <chrono>
#include <iostream>
#include <vector>
struct pow_difficulty_t {
std::chrono::milliseconds timestamp;
int difficulty;
};
int get_valid_difficulty(const std::string& timestamp,
const std::vector<pow_difficulty_t>& history);
bool checkPoW(const std::string& nonce, const std::string& timestamp,
const std::string& ttl, const std::string& recipient,
const std::string& data, std::string& messageHash,
int difficulty);
const int difficulty);

73
pow/src/pow.cpp
View file

@ -13,7 +13,11 @@
#include <sstream>
#include <string.h>
const int BYTE_LEN = 8;
using namespace std::chrono_literals;
constexpr int BYTE_LEN = 8;
constexpr std::chrono::milliseconds TIMESTAMP_VARIANCE = 15min;
using uint64Bytes = std::array<uint8_t, BYTE_LEN>;
// This enforces that the result array has the most significant byte at index 0
@ -35,20 +39,11 @@ bool multWillOverflow(uint64_t left, uint64_t right) {
(std::numeric_limits<std::uint64_t>::max() / left < right);
}
bool checkPoW(const std::string& nonce, const std::string& timestamp,
const std::string& ttl, const std::string& recipient,
const std::string& data, std::string& messageHash,
int difficulty) {
const std::string payload = timestamp + ttl + recipient + data;
bool calcTarget(const std::string& payload, const uint64_t ttlInt,
const int difficulty, uint64Bytes& target) {
bool overflow = addWillOverflow(payload.size(), BYTE_LEN);
if (overflow)
return false;
uint64_t ttlInt;
if (!util::parseTTL(ttl, ttlInt))
return false;
// ttl is in milliseconds, but target calculation wants seconds
ttlInt = ttlInt / 1000;
uint64_t totalLen = payload.size() + BYTE_LEN;
overflow = multWillOverflow(ttlInt, totalLen);
if (overflow)
@ -65,8 +60,60 @@ bool checkPoW(const std::string& nonce, const std::string& timestamp,
uint64_t denominator = difficulty * lenPlusInnerFrac;
uint64_t targetNum = std::numeric_limits<uint64_t>::max() / denominator;
uint64Bytes target;
u64ToU8Array(targetNum, target);
return true;
}
int get_valid_difficulty(const std::string& timestamp,
const std::vector<pow_difficulty_t>& history) {
uint64_t timestamp_long;
try {
timestamp_long = std::stoull(timestamp);
} catch (...) {
// Should never happen, checked previously
return false;
}
const auto msg_timestamp = std::chrono::milliseconds(timestamp_long);
int difficulty = std::numeric_limits<int>::max();
int most_recent_difficulty = std::numeric_limits<int>::max();
std::chrono::milliseconds most_recent(0);
const std::chrono::milliseconds lower = msg_timestamp - TIMESTAMP_VARIANCE;
const std::chrono::milliseconds upper = msg_timestamp + TIMESTAMP_VARIANCE;
for (const auto& this_difficulty : history) {
const std::chrono::milliseconds t = this_difficulty.timestamp;
if (t < msg_timestamp && t >= most_recent) {
most_recent = t;
most_recent_difficulty = this_difficulty.difficulty;
}
if (t >= lower && t <= upper) {
difficulty = std::min(this_difficulty.difficulty, difficulty);
}
}
return std::min(most_recent_difficulty, difficulty);
}
bool checkPoW(const std::string& nonce, const std::string& timestamp,
const std::string& ttl, const std::string& recipient,
const std::string& data, std::string& messageHash,
const int difficulty) {
std::string payload;
payload.reserve(timestamp.size() + ttl.size() + recipient.size() +
data.size());
payload += timestamp;
payload += ttl;
payload += recipient;
payload += data;
uint64_t ttlInt;
if (!util::parseTTL(ttl, ttlInt))
return false;
// ttl is in milliseconds, but target calculation wants seconds
ttlInt = ttlInt / 1000;
uint64Bytes target;
calcTarget(payload, ttlInt, difficulty, target);
uint8_t hashResult[SHA512_DIGEST_LENGTH];
// Initial hash

30
unit_test/pow.cpp
View file

@ -2,6 +2,9 @@
#include "utils.hpp"
#include <boost/test/unit_test.hpp>
#include <chrono>
using namespace std::chrono_literals;
BOOST_AUTO_TEST_SUITE(pow_unit_test)
@ -23,6 +26,8 @@ constexpr auto data =
"wwo52CiwtbrFdXc84K0yDKTnPu6QoTKoiVt+hftgKv0o/"
"ruRN40fLSGl0KD+FqLf4Oqe5u1MaDDoUlLW1tdw6gsPSUxgQPWkJ6qda3t+"
"IioolNQWidLcKg4WXvSwWRO6bDLNXfyDLd9UwcpHRus5UYnmc9BTjiXHQP8pWc4ciCMAQ==";
const std::vector<pow_difficulty_t> history{
pow_difficulty_t{std::chrono::milliseconds(1554559211), 10}};
} // namespace valid_pow
BOOST_AUTO_TEST_CASE(util_parses_a_valid_ttl) {
@ -105,4 +110,29 @@ BOOST_AUTO_TEST_CASE(it_checks_an_invalid_data) {
false);
}
BOOST_AUTO_TEST_CASE(it_checks_correct_difficulty) {
using namespace valid_pow;
std::string messageHash;
const auto t = std::chrono::milliseconds(1554859211);
const auto t1 = t - 30min;
const auto t2 = t - 20min;
const std::vector<pow_difficulty_t> history1{pow_difficulty_t{t1, 1000},
pow_difficulty_t{t2, 10}};
BOOST_CHECK_EQUAL(get_valid_difficulty(timestamp, history1), 10);
const auto t3 = t - 30min;
const auto t4 = t + 5min;
const std::vector<pow_difficulty_t> history2{pow_difficulty_t{t1, 1000},
pow_difficulty_t{t2, 10}};
BOOST_CHECK_EQUAL(get_valid_difficulty(timestamp, history2), 10);
const auto t5 = t - 5min;
const auto t6 = t - 8min;
const std::vector<pow_difficulty_t> history3{pow_difficulty_t{t3, 1000},
pow_difficulty_t{t4, 10}};
BOOST_CHECK_EQUAL(get_valid_difficulty(timestamp, history3), 10);
}
BOOST_AUTO_TEST_SUITE_END()