TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-08-03 17:56:12 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(dwarfs): add block_allocator option to FUSE driver

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2025-04-07 13:09:05 +02:00
parent d739973d00
commit e71cc49353
3 changed files with 36 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
doc/dwarfs.md
View File

@ -189,6 +189,14 @@ options:
be an integer value. Suffixes `ms`, `s`, `m`, `h` are supported.
If no suffix is given, the value will be assumed to be in seconds.
- `-o block_allocator=malloc`|`mmap`:
Select the allocator for decompressed file system blocks. By default,
blocks will be allocated using `malloc`. However, depending on the way
that `malloc` is implemented on your system, you may find that memory
used by `dwarfs` isn't released despite using cache tidying. In this
case, using the `mmap` block allocator is much more likely to release
the memory.
- `-o seq_detector=`*num*:
Threshold, in blocks, for the sequential access detector. If the most
recently accessed *num* blocks are sequential, then the block following

8
test/tools_test.cpp
View File

@ -1037,7 +1037,7 @@ TEST_P(tools_test, end_to_end) {
std::vector<std::string> all_options{
"-s",
"-ocase_insensitive",
"-ocase_insensitive,block_allocator=mmap",
#ifndef _WIN32
"-oenable_nlink",
"-oreadonly",
@ -1067,14 +1067,14 @@ TEST_P(tools_test, end_to_end) {
for (size_t i = 0; i < all_options.size(); ++i) {
if ((1 << i) & bitmask) {
auto const& opt = all_options[i];
if (opt == "-ocase_insensitive") {
if (opt.find("-ocase_insensitive") != std::string::npos) {
case_insensitive = true;
}
#ifndef _WIN32
if (opt == "-oreadonly") {
if (opt.find("-oreadonly") != std::string::npos) {
readonly = true;
}
if (opt == "-oenable_nlink") {
if (opt.find("-oenable_nlink") != std::string::npos) {
enable_nlink = true;
}
if (opt.find("-ouid=") != std::string::npos) {

24
tools/src/dwarfs_main.cpp
View File

@ -182,6 +182,7 @@ struct options {
char const* cache_tidy_strategy_str{nullptr}; // TODO: const?? -> use string?
char const* cache_tidy_interval_str{nullptr}; // TODO: const?? -> use string?
char const* cache_tidy_max_age_str{nullptr}; // TODO: const?? -> use string?
char const* block_alloc_mode_str{nullptr}; // TODO: const?? -> use string?
char const* seq_detector_thresh_str{nullptr}; // TODO: const?? -> use string?
char const* analysis_file_str{nullptr}; // TODO: const?? -> use string?
#ifndef _WIN32
@ -209,6 +210,8 @@ struct options {
reader::cache_tidy_strategy::NONE};
std::chrono::milliseconds block_cache_tidy_interval{std::chrono::minutes(5)};
std::chrono::milliseconds block_cache_tidy_max_age{std::chrono::minutes{10}};
reader::block_cache_allocation_mode block_allocator{
reader::block_cache_allocation_mode::MALLOC};
size_t seq_detector_threshold{kDefaultSeqDetectorThreshold};
#ifndef _WIN32
std::optional<file_stat::uid_type> fs_uid;
@ -339,6 +342,7 @@ constexpr std::array dwarfs_opts{
DWARFS_OPT("tidy_strategy=%s", cache_tidy_strategy_str, 0),
DWARFS_OPT("tidy_interval=%s", cache_tidy_interval_str, 0),
DWARFS_OPT("tidy_max_age=%s", cache_tidy_max_age_str, 0),
DWARFS_OPT("block_allocator=%s", block_alloc_mode_str, 0),
DWARFS_OPT("seq_detector=%s", seq_detector_thresh_str, 0),
DWARFS_OPT("analysis_file=%s", analysis_file_str, 0),
DWARFS_OPT("preload_category=%s", preload_category_str, 0),
@ -363,6 +367,11 @@ constexpr sorted_array_map cache_tidy_strategy_map{
std::pair{"swap"sv, reader::cache_tidy_strategy::BLOCK_SWAPPED_OUT},
};
constexpr sorted_array_map block_allocator_map{
std::pair{"malloc"sv, reader::block_cache_allocation_mode::MALLOC},
std::pair{"mmap"sv, reader::block_cache_allocation_mode::MMAP},
};
constexpr std::string_view pid_xattr{"user.dwarfs.driver.pid"};
constexpr std::string_view perfmon_xattr{"user.dwarfs.driver.perfmon"};
constexpr std::string_view inodeinfo_xattr{"user.dwarfs.inodeinfo"};
@ -1301,6 +1310,7 @@ void usage(std::ostream& os, std::filesystem::path const& progname) {
<< " -o tidy_strategy=NAME (none)|time|swap\n"
<< " -o tidy_interval=TIME interval for cache tidying (5m)\n"
<< " -o tidy_max_age=TIME tidy blocks after this time (10m)\n"
<< " -o block_allocator=NAME (malloc)|mmap\n"
<< " -o seq_detector=NUM sequential access detector threshold (4)\n"
#if DWARFS_PERFMON_ENABLED
<< " -o perfmon=name[+...] enable performance monitor\n"
@ -1526,6 +1536,7 @@ void load_filesystem(dwarfs_userdata& userdata) {
fsopts.block_cache.init_workers = false;
fsopts.block_cache.sequential_access_detector_threshold =
opts.seq_detector_threshold;
fsopts.block_cache.allocation_mode = opts.block_allocator;
fsopts.inode_reader.readahead = opts.readahead;
fsopts.metadata.enable_nlink = bool(opts.enable_nlink);
fsopts.metadata.readonly = bool(opts.readonly);
@ -1737,6 +1748,19 @@ int dwarfs_main(int argc, sys_char** argv, iolayer const& iol) {
return 1;
}
if (opts.block_alloc_mode_str) {
if (auto it = block_allocator_map.find(opts.block_alloc_mode_str);
it != block_allocator_map.end()) {
opts.block_allocator = it->second;
} else {
iol.err << "error: no such block allocator: " << opts.block_alloc_mode_str
<< "\n";
return 1;
}
} else {
opts.block_allocator = reader::block_cache_allocation_mode::MALLOC;
}
opts.seq_detector_threshold = opts.seq_detector_thresh_str
? to<size_t>(opts.seq_detector_thresh_str)
: kDefaultSeqDetectorThreshold;