diff --git a/doc/dwarfs.md b/doc/dwarfs.md index f1718fa6..1abcb7fb 100644 --- a/doc/dwarfs.md +++ b/doc/dwarfs.md @@ -189,6 +189,14 @@ options: be an integer value. Suffixes `ms`, `s`, `m`, `h` are supported. If no suffix is given, the value will be assumed to be in seconds. +- `-o block_allocator=malloc`|`mmap`: + Select the allocator for decompressed file system blocks. By default, + blocks will be allocated using `malloc`. However, depending on the way + that `malloc` is implemented on your system, you may find that memory + used by `dwarfs` isn't released despite using cache tidying. In this + case, using the `mmap` block allocator is much more likely to release + the memory. + - `-o seq_detector=`*num*: Threshold, in blocks, for the sequential access detector. If the most recently accessed *num* blocks are sequential, then the block following diff --git a/test/tools_test.cpp b/test/tools_test.cpp index 89a1f04b..ecab6997 100644 --- a/test/tools_test.cpp +++ b/test/tools_test.cpp @@ -1037,7 +1037,7 @@ TEST_P(tools_test, end_to_end) { std::vector all_options{ "-s", - "-ocase_insensitive", + "-ocase_insensitive,block_allocator=mmap", #ifndef _WIN32 "-oenable_nlink", "-oreadonly", @@ -1067,14 +1067,14 @@ TEST_P(tools_test, end_to_end) { for (size_t i = 0; i < all_options.size(); ++i) { if ((1 << i) & bitmask) { auto const& opt = all_options[i]; - if (opt == "-ocase_insensitive") { + if (opt.find("-ocase_insensitive") != std::string::npos) { case_insensitive = true; } #ifndef _WIN32 - if (opt == "-oreadonly") { + if (opt.find("-oreadonly") != std::string::npos) { readonly = true; } - if (opt == "-oenable_nlink") { + if (opt.find("-oenable_nlink") != std::string::npos) { enable_nlink = true; } if (opt.find("-ouid=") != std::string::npos) { diff --git a/tools/src/dwarfs_main.cpp b/tools/src/dwarfs_main.cpp index 33a3bbb0..668df3df 100644 --- a/tools/src/dwarfs_main.cpp +++ b/tools/src/dwarfs_main.cpp @@ -182,6 +182,7 @@ struct options { char const* cache_tidy_strategy_str{nullptr}; // TODO: const?? -> use string? char const* cache_tidy_interval_str{nullptr}; // TODO: const?? -> use string? char const* cache_tidy_max_age_str{nullptr}; // TODO: const?? -> use string? + char const* block_alloc_mode_str{nullptr}; // TODO: const?? -> use string? char const* seq_detector_thresh_str{nullptr}; // TODO: const?? -> use string? char const* analysis_file_str{nullptr}; // TODO: const?? -> use string? #ifndef _WIN32 @@ -209,6 +210,8 @@ struct options { reader::cache_tidy_strategy::NONE}; std::chrono::milliseconds block_cache_tidy_interval{std::chrono::minutes(5)}; std::chrono::milliseconds block_cache_tidy_max_age{std::chrono::minutes{10}}; + reader::block_cache_allocation_mode block_allocator{ + reader::block_cache_allocation_mode::MALLOC}; size_t seq_detector_threshold{kDefaultSeqDetectorThreshold}; #ifndef _WIN32 std::optional fs_uid; @@ -339,6 +342,7 @@ constexpr std::array dwarfs_opts{ DWARFS_OPT("tidy_strategy=%s", cache_tidy_strategy_str, 0), DWARFS_OPT("tidy_interval=%s", cache_tidy_interval_str, 0), DWARFS_OPT("tidy_max_age=%s", cache_tidy_max_age_str, 0), + DWARFS_OPT("block_allocator=%s", block_alloc_mode_str, 0), DWARFS_OPT("seq_detector=%s", seq_detector_thresh_str, 0), DWARFS_OPT("analysis_file=%s", analysis_file_str, 0), DWARFS_OPT("preload_category=%s", preload_category_str, 0), @@ -363,6 +367,11 @@ constexpr sorted_array_map cache_tidy_strategy_map{ std::pair{"swap"sv, reader::cache_tidy_strategy::BLOCK_SWAPPED_OUT}, }; +constexpr sorted_array_map block_allocator_map{ + std::pair{"malloc"sv, reader::block_cache_allocation_mode::MALLOC}, + std::pair{"mmap"sv, reader::block_cache_allocation_mode::MMAP}, +}; + constexpr std::string_view pid_xattr{"user.dwarfs.driver.pid"}; constexpr std::string_view perfmon_xattr{"user.dwarfs.driver.perfmon"}; constexpr std::string_view inodeinfo_xattr{"user.dwarfs.inodeinfo"}; @@ -1301,6 +1310,7 @@ void usage(std::ostream& os, std::filesystem::path const& progname) { << " -o tidy_strategy=NAME (none)|time|swap\n" << " -o tidy_interval=TIME interval for cache tidying (5m)\n" << " -o tidy_max_age=TIME tidy blocks after this time (10m)\n" + << " -o block_allocator=NAME (malloc)|mmap\n" << " -o seq_detector=NUM sequential access detector threshold (4)\n" #if DWARFS_PERFMON_ENABLED << " -o perfmon=name[+...] enable performance monitor\n" @@ -1526,6 +1536,7 @@ void load_filesystem(dwarfs_userdata& userdata) { fsopts.block_cache.init_workers = false; fsopts.block_cache.sequential_access_detector_threshold = opts.seq_detector_threshold; + fsopts.block_cache.allocation_mode = opts.block_allocator; fsopts.inode_reader.readahead = opts.readahead; fsopts.metadata.enable_nlink = bool(opts.enable_nlink); fsopts.metadata.readonly = bool(opts.readonly); @@ -1737,6 +1748,19 @@ int dwarfs_main(int argc, sys_char** argv, iolayer const& iol) { return 1; } + if (opts.block_alloc_mode_str) { + if (auto it = block_allocator_map.find(opts.block_alloc_mode_str); + it != block_allocator_map.end()) { + opts.block_allocator = it->second; + } else { + iol.err << "error: no such block allocator: " << opts.block_alloc_mode_str + << "\n"; + return 1; + } + } else { + opts.block_allocator = reader::block_cache_allocation_mode::MALLOC; + } + opts.seq_detector_threshold = opts.seq_detector_thresh_str ? to(opts.seq_detector_thresh_str) : kDefaultSeqDetectorThreshold;