TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-12 13:59:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Implement option to mlock metadata

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2020-11-29 02:47:13 +01:00
parent 9c3fb8b00f
commit 1e70a2d4af
10 changed files with 132 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/dwarfs/filesystem_v2.h
View File

@ -42,7 +42,7 @@ struct statvfs;
namespace dwarfs {
struct block_cache_options;
struct filesystem_options;
struct iovec_read_buf;
class filesystem_writer;
@ -55,7 +55,7 @@ class filesystem_v2 {
filesystem_v2(logger& lgr, std::shared_ptr<mmif> mm);
filesystem_v2(logger& lgr, std::shared_ptr<mmif> mm,
const block_cache_options& bc_options,
const filesystem_options& options,
const struct ::stat* stat_defaults = nullptr,
int inode_offset = 0);

7
include/dwarfs/logger.h
View File

@ -31,6 +31,7 @@
#include <sstream>
#include <stdexcept>
#include <string>
#include <string_view>
#include <tuple>
#include <type_traits>
#include <utility>
@ -60,7 +61,7 @@ class logger {
policy_name_ = name;
}
static level_type parse_level(const std::string& level);
static level_type parse_level(std::string_view level);
private:
std::string policy_name_; // TODO: const?
@ -119,14 +120,17 @@ class timed_level_logger {
: data_(std::move(ll.data_)) {}
~timed_level_logger() {
if (data_->output) {
std::chrono::duration<double> sec =
std::chrono::high_resolution_clock::now() - data_->start_time;
data_->oss << " [" << time_with_unit(sec.count()) << "]";
data_->lgr.write(data_->level, data_->oss.str());
}
}
template <typename T>
timed_level_logger& operator<<(const T& val) {
data_->output = true;
data_->oss << val;
return *this;
}
@ -142,6 +146,7 @@ class timed_level_logger {
std::ostringstream oss;
const logger::level_type level;
std::chrono::time_point<std::chrono::high_resolution_clock> start_time;
bool output{false};
};
std::unique_ptr<data> data_;

14
include/dwarfs/options.h
View File

@ -26,18 +26,28 @@
namespace dwarfs {
enum class mlock_mode { NONE, TRY, MUST };
struct block_cache_options {
size_t max_bytes{0};
size_t num_workers{0};
double decompress_ratio{1.0};
};
enum class file_order_mode { NONE, PATH, SCRIPT, SIMILARITY };
struct filesystem_options {
mlock_mode lock_mode{mlock_mode::NONE};
block_cache_options block_cache;
};
std::ostream& operator<<(std::ostream& os, file_order_mode mode);
enum class file_order_mode { NONE, PATH, SCRIPT, SIMILARITY };
struct scanner_options {
file_order_mode file_order;
bool no_time;
};
std::ostream& operator<<(std::ostream& os, file_order_mode mode);
mlock_mode parse_mlock_mode(std::string_view mode);
} // namespace dwarfs

49
src/dwarfs.cpp
View File

@ -50,9 +50,11 @@ struct options {
const char* cachesize_str; // TODO: const?? -> use string?
const char* debuglevel_str; // TODO: const?? -> use string?
const char* workers_str; // TODO: const?? -> use string?
const char* mlock_str; // TODO: const?? -> use string?
const char* decompress_ratio_str; // TODO: const?? -> use string?
size_t cachesize;
size_t workers;
mlock_mode lock_mode;
double decompress_ratio;
logger::level_type debuglevel;
struct ::stat stat_defaults;
@ -68,32 +70,40 @@ struct options {
const struct fuse_opt dwarfs_opts[] = {
// TODO: user, group, atime, mtime, ctime for those fs who don't have it?
// second level cachesize
DWARFS_OPT("cachesize=%s", cachesize_str, 0),
DWARFS_OPT("debuglevel=%s", debuglevel_str, 0),
DWARFS_OPT("workers=%s", workers_str, 0),
DWARFS_OPT("decratio=%s", decompress_ratio_str, 0), FUSE_OPT_END};
DWARFS_OPT("mlock=%s", mlock_str, 0),
DWARFS_OPT("decratio=%s", decompress_ratio_str, 0),
FUSE_OPT_END};
options opts;
stream_logger s_lgr(std::cerr);
std::shared_ptr<filesystem_v2> s_fs;
struct fuse_session* s_session;
void op_init(void* /*userdata*/, struct fuse_conn_info* /*conn*/) {
DEBUG_FUNC("")
log_proxy<debug_logger_policy> log(s_lgr);
try {
auto ti = log.timed_info();
block_cache_options bco;
bco.max_bytes = opts.cachesize;
bco.num_workers = opts.workers;
bco.decompress_ratio = opts.decompress_ratio;
filesystem_options options;
options.lock_mode = opts.lock_mode;
options.block_cache.max_bytes = opts.cachesize;
options.block_cache.num_workers = opts.workers;
options.block_cache.decompress_ratio = opts.decompress_ratio;
s_fs = std::make_shared<filesystem_v2>(
s_lgr, std::make_shared<mmap>(opts.fsimage), bco, &opts.stat_defaults,
FUSE_ROOT_ID);
s_lgr, std::make_shared<mmap>(opts.fsimage), options,
&opts.stat_defaults, FUSE_ROOT_ID);
ti << "file system initialized";
} catch (const std::exception& e) {
log.error() << "error initializing file system: " << e.what();
fuse_session_exit(s_session);
}
}
void op_destroy(void* /*userdata*/) {
@ -376,8 +386,9 @@ void usage(const char* progname) {
<< "DWARFS options:\n"
<< " -o cachesize=SIZE set size of block cache (512M)\n"
<< " -o workers=NUM number of worker threads (2)\n"
<< " -o mlock=NAME mlock mode: (none), try, must\n"
<< " -o decratio=NUM ratio for full decompression (0.8)\n"
<< " -o debuglevel=NAME error, warn, info, debug, trace\n"
<< " -o debuglevel=NAME error, warn, (info), debug, trace\n"
<< std::endl;
fuse_cmdline_help();
@ -438,27 +449,27 @@ int run_fuse(struct fuse_args& args) {
// fsops.getxattr = op_getxattr;
// fsops.listxattr = op_listxattr;
auto se = fuse_session_new(&args, &fsops, sizeof(fsops), nullptr);
s_session = fuse_session_new(&args, &fsops, sizeof(fsops), nullptr);
int err = 1;
if (se) {
if (fuse_set_signal_handlers(se) == 0) {
if (fuse_session_mount(se, fuse_opts.mountpoint) == 0) {
if (s_session) {
if (fuse_set_signal_handlers(s_session) == 0) {
if (fuse_session_mount(s_session, fuse_opts.mountpoint) == 0) {
if (fuse_daemonize(fuse_opts.foreground) == 0) {
if (fuse_opts.singlethread) {
err = fuse_session_loop(se);
err = fuse_session_loop(s_session);
} else {
struct fuse_loop_config config;
config.clone_fd = fuse_opts.clone_fd;
config.max_idle_threads = fuse_opts.max_idle_threads;
err = fuse_session_loop_mt(se, &config);
err = fuse_session_loop_mt(s_session, &config);
}
}
fuse_session_unmount(se);
fuse_session_unmount(s_session);
}
fuse_remove_signal_handlers(se);
fuse_remove_signal_handlers(s_session);
}
fuse_session_destroy(se);
fuse_session_destroy(s_session);
}
::free(fuse_opts.mountpoint);
@ -485,6 +496,8 @@ int main(int argc, char* argv[]) {
? logger::parse_level(opts.debuglevel_str)
: logger::INFO;
opts.workers = opts.workers_str ? folly::to<size_t>(opts.workers_str) : 2;
opts.lock_mode =
opts.mlock_str ? parse_mlock_mode(opts.mlock_str) : mlock_mode::NONE;
opts.decompress_ratio = opts.decompress_ratio_str
? folly::to<double>(opts.decompress_ratio_str)
: 0.8;

40
src/dwarfs/filesystem_v2.cpp
View File

@ -19,6 +19,7 @@
* along with dwarfs. If not, see <https://www.gnu.org/licenses/>.
*/
#include <cerrno>
#include <cstddef>
#include <cstring>
#include <optional>
@ -26,8 +27,11 @@
#include <unordered_map>
#include <vector>
#include <sys/mman.h>
#include <sys/statvfs.h>
#include <boost/system/system_error.hpp>
#include <folly/Range.h>
#include <fmt/format.h>
@ -141,7 +145,9 @@ make_metadata(logger& lgr, std::shared_ptr<mmif> mm,
section_map const& sections, std::vector<uint8_t>& schema_buffer,
std::vector<uint8_t>& meta_buffer,
const struct ::stat* stat_defaults = nullptr,
int inode_offset = 0, bool force_buffers = false) {
int inode_offset = 0, bool force_buffers = false,
mlock_mode lock_mode = mlock_mode::NONE) {
log_proxy<debug_logger_policy> log(lgr);
auto schema_it = sections.find(section_type::METADATA_V2_SCHEMA);
auto meta_it = sections.find(section_type::METADATA_V2);
@ -153,18 +159,32 @@ make_metadata(logger& lgr, std::shared_ptr<mmif> mm,
throw std::runtime_error("no metadata found");
}
auto meta_section =
get_section_data(mm, meta_it->second, meta_buffer, force_buffers);
if (lock_mode != mlock_mode::NONE) {
int rv = ::mlock(meta_section.data(), meta_section.size());
if (rv != 0) {
boost::system::error_code ec(errno, boost::system::generic_category());
if (lock_mode == mlock_mode::MUST) {
throw boost::system::system_error(ec, "mlock");
} else {
log.warn() << "mlock() failed: " << ec.message();
}
}
}
return metadata_v2(
lgr,
get_section_data(mm, schema_it->second, schema_buffer, force_buffers),
get_section_data(mm, meta_it->second, meta_buffer, force_buffers),
stat_defaults, inode_offset);
meta_section, stat_defaults, inode_offset);
}
template <typename LoggerPolicy>
class filesystem_ : public filesystem_v2::impl {
public:
filesystem_(logger& lgr_, std::shared_ptr<mmif> mm,
const block_cache_options& bc_options,
const filesystem_options& options,
const struct ::stat* stat_defaults, int inode_offset);
void dump(std::ostream& os, int detail_level) const override;
@ -198,13 +218,13 @@ class filesystem_ : public filesystem_v2::impl {
template <typename LoggerPolicy>
filesystem_<LoggerPolicy>::filesystem_(logger& lgr, std::shared_ptr<mmif> mm,
const block_cache_options& bc_options,
const filesystem_options& options,
const struct ::stat* stat_defaults,
int inode_offset)
: log_(lgr)
, mm_(mm) {
filesystem_parser parser(mm_);
block_cache cache(lgr, bc_options);
block_cache cache(lgr, options.block_cache);
section_map sections;
@ -223,7 +243,7 @@ filesystem_<LoggerPolicy>::filesystem_(logger& lgr, std::shared_ptr<mmif> mm,
std::vector<uint8_t> schema_buffer;
meta_ = make_metadata(lgr, mm_, sections, schema_buffer, meta_buffer_,
stat_defaults, inode_offset);
stat_defaults, inode_offset, false, options.lock_mode);
log_.debug() << "read " << cache.block_count() << " blocks and "
<< meta_.size() << " bytes of metadata";
@ -341,15 +361,15 @@ ssize_t filesystem_<LoggerPolicy>::readv(uint32_t inode, iovec_read_buf& buf,
} // namespace
filesystem_v2::filesystem_v2(logger& lgr, std::shared_ptr<mmif> mm)
: filesystem_v2(lgr, std::move(mm), block_cache_options()) {}
: filesystem_v2(lgr, std::move(mm), filesystem_options()) {}
filesystem_v2::filesystem_v2(logger& lgr, std::shared_ptr<mmif> mm,
const block_cache_options& bc_options,
const filesystem_options& options,
const struct ::stat* stat_defaults,
int inode_offset)
: impl_(make_unique_logging_object<filesystem_v2::impl, filesystem_,
logger_policies>(
lgr, std::move(mm), bc_options, stat_defaults, inode_offset)) {}
lgr, std::move(mm), options, stat_defaults, inode_offset)) {}
void filesystem_v2::rewrite(logger& lgr, progress& prog,
std::shared_ptr<mmif> mm,

21
src/dwarfs/logger.cpp
View File

@ -29,6 +29,8 @@
#include <boost/date_time/posix_time/posix_time.hpp>
#include <fmt/format.h>
#include "dwarfs/logger.h"
namespace dwarfs {
@ -69,18 +71,23 @@ void backtrace(std::ostream& os) {
}
} // namespace
logger::level_type logger::parse_level(const std::string& level) {
if (level == "error")
logger::level_type logger::parse_level(std::string_view level) {
if (level == "error") {
return ERROR;
if (level == "warn")
}
if (level == "warn") {
return WARN;
if (level == "info")
}
if (level == "info") {
return INFO;
if (level == "debug")
}
if (level == "debug") {
return DEBUG;
if (level == "trace")
}
if (level == "trace") {
return TRACE;
throw std::runtime_error("invalid logger level");
}
throw std::runtime_error(fmt::format("invalid logger level: {}", level));
}
stream_logger::stream_logger(std::ostream& os, level_type threshold)

1
src/dwarfs/mmap.cpp
View File

@ -26,6 +26,7 @@
#include <sys/stat.h>
#include <unistd.h>
#include <boost/system/error_code.hpp>
#include <boost/system/system_error.hpp>
#include "dwarfs/mmap.h"

15
src/dwarfs/options.cpp
View File

@ -22,6 +22,8 @@
#include <ostream>
#include <string>
#include <fmt/format.h>
#include "dwarfs/options.h"
namespace dwarfs {
@ -49,4 +51,17 @@ std::ostream& operator<<(std::ostream& os, file_order_mode mode) {
return os << modestr;
}
mlock_mode parse_mlock_mode(std::string_view mode) {
if (mode == "none") {
return mlock_mode::NONE;
}
if (mode == "try") {
return mlock_mode::TRY;
}
if (mode == "must") {
return mlock_mode::MUST;
}
throw std::runtime_error(fmt::format("invalid lock mode: {}", mode));
}
} // namespace dwarfs

17
src/dwarfsbench.cpp
View File

@ -39,7 +39,8 @@ using namespace dwarfs;
namespace {
int dwarfsbench(int argc, char** argv) {
std::string filesystem, cache_size_str, decompress_ratio_str, log_level;
std::string filesystem, cache_size_str, lock_mode_str, decompress_ratio_str,
log_level;
size_t num_workers;
size_t num_readers;
@ -58,6 +59,9 @@ int dwarfsbench(int argc, char** argv) {
("cache-size,s",
po::value<std::string>(&cache_size_str)->default_value("256m"),
"block cache size")
("lock-mode,m",
po::value<std::string>(&cache_size_str)->default_value("none"),
"mlock mode (none, try, must)")
("decompress-ratio,r",
po::value<std::string>(&decompress_ratio_str)->default_value("0.8"),
"block cache size")
@ -80,14 +84,15 @@ int dwarfsbench(int argc, char** argv) {
}
stream_logger lgr(std::cerr, logger::parse_level(log_level));
block_cache_options bco;
filesystem_options fsopts;
bco.max_bytes = parse_size_with_unit(cache_size_str);
bco.num_workers = num_workers;
bco.decompress_ratio = folly::to<double>(decompress_ratio_str);
fsopts.lock_mode = parse_mlock_mode(lock_mode_str);
fsopts.block_cache.max_bytes = parse_size_with_unit(cache_size_str);
fsopts.block_cache.num_workers = num_workers;
fsopts.block_cache.decompress_ratio = folly::to<double>(decompress_ratio_str);
dwarfs::filesystem_v2 fs(lgr, std::make_shared<dwarfs::mmap>(filesystem),
bco);
fsopts);
worker_group wg("reader", num_readers);

6
test/dwarfs.cpp
View File

@ -196,10 +196,10 @@ void basic_end_to_end_test(const std::string& compressor,
auto mm = std::make_shared<test::mmap_mock>(oss.str());
block_cache_options bco;
bco.max_bytes = 1 << 20;
filesystem_options opts;
opts.block_cache.max_bytes = 1 << 20;
filesystem_v2 fs(lgr, mm, bco);
filesystem_v2 fs(lgr, mm, opts);
auto entry = fs.find("/foo.pl");
struct ::stat st;