1
1
Fork 0
mirror of https://github.com/oxen-io/lokinet synced 2023-12-14 06:53:00 +01:00

Add binary file slurp/dump utility functions

We have basically this same bit of code in tons of places; consolidate
it into llarp::util::slurp_file/llarp::util::dump_file.

Also renames all the extra junk that crept into llarp/util/fs.hpp out of
there into llarp/util/file.hpp instead.
This commit is contained in:
Jason Rhinelander 2022-10-05 15:05:25 -03:00
parent 8b321612da
commit d335527a70
No known key found for this signature in database
GPG key ID: C4992CE7A88D4262
13 changed files with 349 additions and 285 deletions

View file

@ -5,7 +5,7 @@ add_library(lokinet-util
${CMAKE_CURRENT_BINARY_DIR}/constants/version.cpp
util/bencode.cpp
util/buffer.cpp
util/fs.cpp
util/file.cpp
util/json.cpp
util/logging/buffer.cpp
util/easter_eggs.cpp

View file

@ -10,7 +10,7 @@
#include <llarp/net/ip.hpp>
#include <llarp/router_contact.hpp>
#include <stdexcept>
#include <llarp/util/fs.hpp>
#include <llarp/util/file.hpp>
#include <llarp/util/formattable.hpp>
#include <llarp/util/logging.hpp>
#include <llarp/util/mem.hpp>
@ -19,7 +19,6 @@
#include <llarp/service/name.hpp>
#include <cstdlib>
#include <fstream>
#include <ios>
#include <iostream>
@ -1434,18 +1433,19 @@ namespace llarp
bool
Config::Load(std::optional<fs::path> fname, bool isRelay)
{
std::vector<char> ini{};
std::string ini;
if (fname)
{
if (not fs::exists(*fname))
try
{
ini = util::slurp_file(*fname);
}
catch (const std::exception&)
{
return false;
fs::ifstream inf{*fname, std::ios::in | std::ios::binary};
auto sz = inf.seekg(0, std::ios::end).tellg();
inf.seekg(0, std::ios::beg);
ini.resize(sz);
inf.read(ini.data(), ini.size());
}
}
return LoadConfigData(std::string_view{ini.data(), ini.size()}, fname, isRelay);
return LoadConfigData(ini, fname, isRelay);
}
bool
@ -1521,12 +1521,15 @@ namespace llarp
confStr = config.generateBaseClientConfig();
// open a filestream
auto stream = llarp::util::OpenFileStream<std::ofstream>(confFile.c_str(), std::ios::binary);
if (not stream or not stream->is_open())
throw std::runtime_error{fmt::format("Failed to open file {} for writing", confFile)};
*stream << confStr;
stream->flush();
try
{
util::dump_file(confFile, confStr);
}
catch (const std::exception& e)
{
throw std::runtime_error{
fmt::format("Failed to write config data to {}: {}", confFile, e.what())};
}
llarp::LogInfo("Generated new config ", confFile);
}

View file

@ -13,19 +13,19 @@
namespace llarp
{
bool
ConfigParser::LoadFile(const fs::path fname)
ConfigParser::LoadFile(const fs::path& fname)
{
try
{
std::ifstream f(fname, std::ios::in | std::ios::binary);
if (not f.is_open())
return false;
f.seekg(0, std::ios::end);
m_Data.resize(f.tellg());
f.seekg(0, std::ios::beg);
if (m_Data.size() == 0)
return false;
f.read(m_Data.data(), m_Data.size());
m_Data = util::slurp_file(fname);
}
catch (const std::exception& e)
{
return false;
}
if (m_Data.empty())
return false;
m_FileName = fname;
return Parse();
}

View file

@ -6,7 +6,7 @@
#include <memory>
#include <unordered_map>
#include <vector>
#include <llarp/util/fs.hpp>
#include <llarp/util/file.hpp>
namespace llarp
{
@ -22,7 +22,7 @@ namespace llarp
/// return true on success
/// return false on error
bool
LoadFile(const fs::path fname);
LoadFile(const fs::path& fname);
/// load from string
/// return true on success
@ -57,7 +57,7 @@ namespace llarp
bool
Parse();
std::vector<char> m_Data;
std::string m_Data;
Config_impl_t m_Config;
std::unordered_map<fs::path, Config_impl_t, util::FileHash> m_Overrides;
fs::path m_FileName;

View file

@ -1,9 +1,7 @@
#include "types.hpp"
#include <llarp/util/buffer.hpp>
#include <fstream>
#include <llarp/util/fs.hpp>
#include <llarp/util/file.hpp>
#include <iterator>
@ -33,29 +31,25 @@ namespace llarp
bool
SecretKey::LoadFromFile(const fs::path& fname)
{
std::ifstream f(fname.string(), std::ios::in | std::ios::binary);
if (!f.is_open())
size_t sz;
std::array<byte_t, 128> tmp;
try
{
sz = util::slurp_file(fname, tmp.data(), tmp.size());
}
catch (const std::exception&)
{
return false;
}
f.seekg(0, std::ios::end);
const size_t sz = f.tellg();
f.seekg(0, std::ios::beg);
if (sz == size())
{
// is raw buffer
std::copy_n(std::istreambuf_iterator<char>(f), sz, begin());
std::copy_n(tmp.begin(), sz, begin());
return true;
}
std::array<byte_t, 128> tmp;
llarp_buffer_t buf(tmp);
if (sz > sizeof(tmp))
{
return false;
}
f.read(reinterpret_cast<char*>(tmp.data()), sz);
return BDecode(&buf);
}
@ -95,38 +89,43 @@ namespace llarp
bool
SecretKey::SaveToFile(const fs::path& fname) const
{
std::array<byte_t, 128> tmp;
std::string tmp(128, 0);
llarp_buffer_t buf(tmp);
if (!BEncode(&buf))
return false;
tmp.resize(buf.cur - buf.base);
try
{
util::dump_file(fname, tmp);
}
catch (const std::exception&)
{
return false;
}
auto optional_f = llarp::util::OpenFileStream<std::ofstream>(fname, std::ios::binary);
if (!optional_f)
return false;
auto& f = *optional_f;
if (!f.is_open())
return false;
f.write((char*)buf.base, buf.cur - buf.base);
return f.good();
return true;
}
bool
IdentitySecret::LoadFromFile(const fs::path& fname)
{
auto optional = util::OpenFileStream<std::ifstream>(fname, std::ios::binary | std::ios::in);
if (!optional)
return false;
auto& f = *optional;
f.seekg(0, std::ios::end);
const size_t sz = f.tellg();
f.seekg(0, std::ios::beg);
if (sz != 32)
std::array<byte_t, SIZE> buf;
size_t sz;
try
{
llarp::LogError("service node seed size invalid: ", sz, " != 32");
sz = util::slurp_file(fname, buf.data(), buf.size());
}
catch (const std::exception& e)
{
llarp::LogError("failed to load service node seed: ", e.what());
return false;
}
std::copy_n(std::istreambuf_iterator<char>(f), sz, begin());
if (sz != SIZE)
{
llarp::LogError("service node seed size invalid: ", sz, " != ", SIZE);
return false;
}
std::copy(buf.begin(), buf.end(), begin());
return true;
}

View file

@ -12,7 +12,6 @@
#include "dht/kademlia.hpp"
#include <algorithm>
#include <fstream>
#include <unordered_map>
#include <utility>

View file

@ -11,11 +11,12 @@
#include <oxenc/bt_serialize.h>
#include <fstream>
#include "util/fs.hpp"
#include "util/file.hpp"
namespace llarp
{
static auto logcat = log::Cat("RC");
NetID&
NetID::DefaultValue()
{
@ -290,13 +291,13 @@ namespace llarp
}
else
{
llarp::LogWarn("Received RouterContact with unkown version (", outer_version, ")");
log::warning(logcat, "Received RouterContact with unkown version ({})", outer_version);
return false;
}
}
catch (const std::exception& e)
{
llarp::LogDebug("RouterContact::BDecode failed, reason: ", e.what());
log::debug(logcat, "RouterContact::BDecode failed: {}", e.what());
}
return false;
@ -316,14 +317,14 @@ namespace llarp
if (not btlist.is_finished())
{
llarp::LogDebug("RouterContact serialized list too long for specified version.");
log::debug(logcat, "RouterContact serialized list too long for specified version.");
return false;
}
llarp_buffer_t sigbuf(signature_string.data(), signature_string.size());
if (not signature.FromBytestring(&sigbuf))
{
llarp::LogDebug("RouterContact serialized signature had invalid length.");
log::debug(logcat, "RouterContact serialized signature had invalid length.");
return false;
}
@ -477,8 +478,8 @@ namespace llarp
{
if (netID != NetID::DefaultValue())
{
llarp::LogError(
"netid mismatch: '", netID, "' (theirs) != '", NetID::DefaultValue(), "' (ours)");
log::error(
logcat, "netid mismatch: '{}' (theirs) != '{}' (ours)", netID, NetID::DefaultValue());
return false;
}
@ -491,13 +492,13 @@ namespace llarp
{
if (net->IsBogon(a.ip) && BlockBogons)
{
llarp::LogError("invalid address info: ", a);
log::error(logcat, "invalid address info: {}", a);
return false;
}
}
if (!VerifySignature())
{
llarp::LogError("invalid signature: ", *this);
log::error(logcat, "invalid signature: {}", *this);
return false;
}
return true;
@ -515,7 +516,7 @@ namespace llarp
llarp_buffer_t buf(tmp);
if (!copy.BEncode(&buf))
{
llarp::LogError("bencode failed");
log::error(logcat, "bencode failed");
return false;
}
buf.sz = buf.cur - buf.base;
@ -541,18 +542,15 @@ namespace llarp
{
return false;
}
buf.sz = buf.cur - buf.base;
buf.cur = buf.base;
auto f = llarp::util::OpenFileStream<std::ofstream>(fname, std::ios::binary);
if (!f)
try
{
util::dump_file(fname, tmp.data(), buf.cur - buf.base);
}
catch (const std::exception& e)
{
log::error(logcat, "Failed to write RC to {}: {}", fname, e.what());
return false;
}
if (!f->is_open())
{
return false;
}
f->write((char*)buf.base, buf.sz);
return true;
}
@ -561,21 +559,15 @@ namespace llarp
{
std::array<byte_t, MAX_RC_SIZE> tmp;
llarp_buffer_t buf(tmp);
std::ifstream f;
f.open(fname.string(), std::ios::binary);
if (!f.is_open())
try
{
llarp::LogError("Failed to open ", fname);
util::slurp_file(fname, tmp.data(), tmp.size());
}
catch (const std::exception& e)
{
log::error(logcat, "Failed to read RC from {}: {}", fname, e.what());
return false;
}
f.seekg(0, std::ios::end);
auto l = f.tellg();
if (l > static_cast<std::streamoff>(sizeof tmp))
{
return false;
}
f.seekg(0, std::ios::beg);
f.read((char*)tmp.data(), l);
return BDecode(&buf);
}

View file

@ -98,17 +98,16 @@ namespace llarp
RegenerateKeys();
if (!BEncode(&buf))
throw std::length_error("failed to encode new identity");
// rewind
buf.sz = buf.cur - buf.base;
buf.cur = buf.base;
const auto sz = buf.cur - buf.base;
// write
auto optional_f = util::OpenFileStream<std::ofstream>(fname, std::ios::binary);
if (!optional_f)
throw std::runtime_error{fmt::format("can not open {}", fname)};
auto& f = *optional_f;
if (!f.is_open())
throw std::runtime_error{fmt::format("did not open {}", fname)};
f.write((char*)buf.cur, buf.sz);
try
{
util::dump_file(fname, tmp.data(), sz);
}
catch (const std::exception& e)
{
throw std::runtime_error{fmt::format("failed to write {}: {}", fname, e.what())};
}
}
if (not fs::is_regular_file(fname))
@ -117,17 +116,18 @@ namespace llarp
}
// read file
std::ifstream inf(fname, std::ios::binary);
inf.seekg(0, std::ios::end);
size_t sz = inf.tellg();
inf.seekg(0, std::ios::beg);
try
{
util::slurp_file(fname, tmp.data(), tmp.size());
}
catch (const std::length_error&)
{
throw std::length_error{"service identity too big"};
}
// (don't catch io error exceptions)
if (sz > sizeof(tmp))
throw std::length_error("service identity too big");
// decode
inf.read((char*)buf.base, sz);
if (!bencode_decode_dict(*this, &buf))
throw std::length_error("could not decode service identity");
throw std::length_error{"could not decode service identity"};
auto crypto = CryptoManager::instance();

View file

@ -2,12 +2,11 @@
#include "buffer.hpp"
#include "bencode.h"
#include "fs.hpp"
#include "file.hpp"
#include <llarp/util/logging.hpp>
#include "mem.hpp"
#include <type_traits>
#include <fstream>
#include <set>
#include <vector>
@ -338,21 +337,16 @@ namespace llarp
bool
BDecodeReadFile(const fs::path fpath, T& t)
{
std::vector<byte_t> ptr;
std::string content;
try
{
std::ifstream f;
f.open(fpath.string());
if (!f.is_open())
{
return false;
}
f.seekg(0, std::ios::end);
const std::streampos sz = f.tellg();
f.seekg(0, std::ios::beg);
ptr.resize(sz);
f.read((char*)ptr.data(), sz);
content = util::slurp_file(fpath);
}
llarp_buffer_t buf(ptr);
catch (const std::exception&)
{
return false;
}
llarp_buffer_t buf(content);
return t.BDecode(&buf);
}
@ -361,16 +355,18 @@ namespace llarp
bool
BEncodeWriteFile(const fs::path fpath, const T& t)
{
std::array<byte_t, bufsz> tmp;
std::string tmp(bufsz, 0);
llarp_buffer_t buf(tmp);
if (!t.BEncode(&buf))
return false;
buf.sz = buf.cur - buf.base;
tmp.resize(buf.cur - buf.base);
try
{
auto f = llarp::util::OpenFileStream<std::ofstream>(fpath, std::ios::binary);
if (not f or not f->is_open())
return false;
f->write((char*)buf.base, buf.sz);
util::dump_file(fpath, tmp);
}
catch (const std::exception& e)
{
return false;
}
return true;
}

119
llarp/util/file.cpp Normal file
View file

@ -0,0 +1,119 @@
#include "file.hpp"
#include <fstream>
#include <ios>
#include <stdexcept>
#include <llarp/util/logging.hpp>
#include <llarp/util/formattable.hpp>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <system_error>
#ifdef WIN32
#include <io.h>
#else
#include <unistd.h>
#endif
namespace llarp::util
{
static std::pair<fs::ifstream, std::streampos>
slurp_file_open(const fs::path& filename)
{
std::pair<fs::ifstream, std::streampos> f;
auto& [in, size] = f;
in.exceptions(std::ifstream::failbit | std::ifstream::badbit);
in.open(filename, std::ios::binary | std::ios::in);
in.seekg(0, std::ios::end);
size = in.tellg();
in.seekg(0, std::ios::beg);
return f;
}
std::string
slurp_file(const fs::path& filename)
{
std::string contents;
auto [in, size] = slurp_file_open(filename);
contents.resize(size);
in.read(contents.data(), size);
return contents;
}
size_t
slurp_file(const fs::path& filename, char* buffer, size_t buffer_size)
{
auto [in, size] = slurp_file_open(filename);
if (static_cast<size_t>(size) > buffer_size)
throw std::length_error{"file is too large for buffer"};
in.read(buffer, size);
return size;
}
void
dump_file(const fs::path& filename, std::string_view contents)
{
fs::ofstream out;
out.exceptions(std::ifstream::failbit | std::ifstream::badbit);
out.open(filename, std::ios::binary | std::ios::out | std::ios::trunc);
out.write(contents.data(), static_cast<std::streamsize>(contents.size()));
}
static std::error_code
errno_error()
{
int e = errno;
errno = 0;
return std::make_error_code(static_cast<std::errc>(e));
}
error_code_t
EnsurePrivateFile(fs::path pathname)
{
errno = 0;
error_code_t ec = errno_error();
const auto str = pathname.string();
if (fs::exists(pathname, ec)) // file exists
{
auto st = fs::status(pathname);
auto perms = st.permissions();
if ((perms & fs::perms::others_exec) != fs::perms::none)
perms = perms ^ fs::perms::others_exec;
if ((perms & fs::perms::others_write) != fs::perms::none)
perms = perms ^ fs::perms::others_write;
if ((perms & fs::perms::others_write) != fs::perms::none)
perms = perms ^ fs::perms::others_write;
if ((perms & fs::perms::group_read) != fs::perms::none)
perms = perms ^ fs::perms::group_read;
if ((perms & fs::perms::others_read) != fs::perms::none)
perms = perms ^ fs::perms::others_read;
if ((perms & fs::perms::owner_exec) != fs::perms::none)
perms = perms ^ fs::perms::owner_exec;
fs::permissions(pathname, perms, ec);
if (ec)
llarp::LogError("failed to set permissions on ", pathname);
}
else // file is not there
{
errno = 0;
int fd = ::open(str.c_str(), O_RDWR | O_CREAT, 0600);
ec = errno_error();
if (fd != -1)
{
::close(fd);
}
}
#ifndef WIN32
if (ec)
llarp::LogError("failed to ensure ", str, ", ", ec.message());
return ec;
#else
return {};
#endif
}
} // namespace llarp::util

103
llarp/util/file.hpp Normal file
View file

@ -0,0 +1,103 @@
#pragma once
#include "fs.hpp"
#include <optional>
#include <set>
#include <string>
#include <string_view>
#ifndef _MSC_VER
#include <dirent.h>
#endif
namespace llarp::util
{
/// Reads a binary file from disk into a string. Throws on error.
std::string
slurp_file(const fs::path& filename);
/// Reads a binary file from disk directly into a buffer. Throws a std::length_error if the
/// file is bigger than the buffer. Returns the bytes copied on success.
size_t
slurp_file(const fs::path& filename, char* buffer, size_t buffer_size);
/// Same, but for some non-char but single-byte char type (e.g. byte_t, std::byte, unsigned char).
template <
typename Char,
std::enable_if_t<sizeof(Char) == 1 and not std::is_same_v<Char, char>, int> = 1>
inline size_t
slurp_file(const fs::path& filename, Char* buffer, size_t buffer_size)
{
return slurp_file(filename, reinterpret_cast<char*>(buffer), buffer_size);
}
/// Dumps binary string contents to disk. The file is overwritten if it already exists. Throws
/// on error.
void
dump_file(const fs::path& filename, std::string_view contents);
/// Same as above, but works via char-like buffer
template <typename Char, std::enable_if_t<sizeof(Char) == 1, int> = 0>
inline void
dump_file(const fs::path& filename, const Char* buffer, size_t buffer_size)
{
return dump_file(
filename, std::string_view{reinterpret_cast<const char*>(buffer), buffer_size});
}
struct FileHash
{
size_t
operator()(const fs::path& f) const
{
std::hash<std::string> h;
return h(f.string());
}
};
using error_code_t = std::error_code;
/// Ensure that a file exists and has correct permissions
/// return any error code or success
error_code_t
EnsurePrivateFile(fs::path pathname);
/// open a stream to a file and ensure it exists before open
/// sets any permissions on creation
template <typename T>
std::optional<T>
OpenFileStream(fs::path pathname, std::ios::openmode mode)
{
if (EnsurePrivateFile(pathname))
return {};
return std::make_optional<T>(pathname, mode);
}
template <typename PathVisitor>
static void
IterDir(const fs::path& path, PathVisitor visit)
{
DIR* d = opendir(path.string().c_str());
if (d == nullptr)
return;
struct dirent* ent = nullptr;
std::set<fs::path> entries;
do
{
ent = readdir(d);
if (not ent)
break;
if (ent->d_name[0] == '.')
continue;
entries.emplace(path / fs::path{ent->d_name});
} while (ent);
closedir(d);
for (const auto& p : entries)
{
if (not visit(p))
return;
}
}
} // namespace llarp::util

View file

@ -1,76 +0,0 @@
#include "fs.hpp"
#include <llarp/util/logging.hpp>
#include <llarp/util/formattable.hpp>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <system_error>
#ifdef WIN32
#include <io.h>
#else
#include <unistd.h>
#endif
namespace llarp
{
namespace util
{
static std::error_code
errno_error()
{
int e = errno;
errno = 0;
return std::make_error_code(static_cast<std::errc>(e));
}
error_code_t
EnsurePrivateFile(fs::path pathname)
{
errno = 0;
error_code_t ec = errno_error();
const auto str = pathname.string();
if (fs::exists(pathname, ec)) // file exists
{
auto st = fs::status(pathname);
auto perms = st.permissions();
if ((perms & fs::perms::others_exec) != fs::perms::none)
perms = perms ^ fs::perms::others_exec;
if ((perms & fs::perms::others_write) != fs::perms::none)
perms = perms ^ fs::perms::others_write;
if ((perms & fs::perms::others_write) != fs::perms::none)
perms = perms ^ fs::perms::others_write;
if ((perms & fs::perms::group_read) != fs::perms::none)
perms = perms ^ fs::perms::group_read;
if ((perms & fs::perms::others_read) != fs::perms::none)
perms = perms ^ fs::perms::others_read;
if ((perms & fs::perms::owner_exec) != fs::perms::none)
perms = perms ^ fs::perms::owner_exec;
fs::permissions(pathname, perms, ec);
if (ec)
llarp::LogError("failed to set permissions on ", pathname);
}
else // file is not there
{
errno = 0;
int fd = ::open(str.c_str(), O_RDWR | O_CREAT, 0600);
ec = errno_error();
if (fd != -1)
{
::close(fd);
}
}
#ifndef WIN32
if (ec)
llarp::LogError("failed to ensure ", str, ", ", ec.message());
return ec;
#else
return {};
#endif
}
} // namespace util
} // namespace llarp

View file

@ -1,8 +1,5 @@
#pragma once
#include <functional>
#include <set>
#ifdef USE_GHC_FILESYSTEM
#include <ghc/filesystem.hpp>
namespace fs = ghc::filesystem;
@ -15,72 +12,4 @@ namespace fs
using ofstream = std::ofstream;
using fstream = std::fstream;
} // namespace fs
#endif
#ifndef _MSC_VER
#include <dirent.h>
#endif
#include <optional>
namespace llarp
{
namespace util
{
struct FileHash
{
size_t
operator()(const fs::path& f) const
{
std::hash<std::string> h;
return h(f.string());
}
};
using error_code_t = std::error_code;
/// Ensure that a file exists and has correct permissions
/// return any error code or success
error_code_t
EnsurePrivateFile(fs::path pathname);
/// open a stream to a file and ensure it exists before open
/// sets any permissions on creation
template <typename T>
std::optional<T>
OpenFileStream(fs::path pathname, std::ios::openmode mode)
{
if (EnsurePrivateFile(pathname))
return {};
return std::make_optional<T>(pathname, mode);
}
template <typename PathVisitor>
static void
IterDir(const fs::path& path, PathVisitor visit)
{
DIR* d = opendir(path.string().c_str());
if (d == nullptr)
return;
struct dirent* ent = nullptr;
std::set<fs::path> entries;
do
{
ent = readdir(d);
if (not ent)
break;
if (ent->d_name[0] == '.')
continue;
entries.emplace(path / fs::path{ent->d_name});
} while (ent);
closedir(d);
for (const auto& p : entries)
{
if (not visit(p))
return;
}
};
} // namespace util
} // namespace llarp