TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-14 06:48:39 -04:00
Code Issues Packages Projects Releases Wiki Activity

Allow separate configuration of scanner workers

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2020-11-24 22:25:00 +01:00
parent 1f7380fe8f
commit a9e203cf62
1 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/mkdwarfs.cpp
View File

@ -218,7 +218,7 @@ int mkdwarfs(int argc, char** argv) {
block_manager::config cfg; block_manager::config cfg;
std::string path, output, window_sizes, memory_limit, script_path, std::string path, output, window_sizes, memory_limit, script_path,
compression, log_level; compression, log_level;
size_t num_workers; size_t num_workers, num_scanner_workers;
bool no_time = false, no_owner = false, recompress = false, bool no_time = false, no_owner = false, recompress = false,
no_progress = false; no_progress = false;
unsigned level; unsigned level;
@ -245,7 +245,10 @@ int mkdwarfs(int argc, char** argv) {
"block size bits (size = 2^bits)") "block size bits (size = 2^bits)")
("num-workers,N", ("num-workers,N",
po::value<size_t>(&num_workers)->default_value(num_cpu), po::value<size_t>(&num_workers)->default_value(num_cpu),
"number of worker threads") "number of writer worker threads")
("num-scanner-workers,M",
po::value<size_t>(&num_scanner_workers)->default_value(num_cpu),
"number of scanner worker threads")
("memory-limit,L", ("memory-limit,L",
po::value<std::string>(&memory_limit)->default_value("1g"), po::value<std::string>(&memory_limit)->default_value("1g"),
"block manager memory limit") "block manager memory limit")
@ -378,7 +381,8 @@ int mkdwarfs(int argc, char** argv) {
}); });
} }
worker_group wg("writer", num_workers); worker_group wg_writer("writer", num_workers);
worker_group wg_scanner("scanner", num_scanner_workers);
console_writer lgr(std::cerr, !no_progress && ::isatty(::fileno(stderr)), console_writer lgr(std::cerr, !no_progress && ::isatty(::fileno(stderr)),
get_term_width(), logger::parse_level(log_level)); get_term_width(), logger::parse_level(log_level));
@ -402,15 +406,15 @@ int mkdwarfs(int argc, char** argv) {
block_compressor bc(compression); block_compressor bc(compression);
std::ofstream ofs(output); std::ofstream ofs(output);
filesystem_writer fsw(ofs, lgr, wg, prog, bc, mem_limit); filesystem_writer fsw(ofs, lgr, wg_writer, prog, bc, mem_limit);
if (recompress) { if (recompress) {
auto ti = log.timed_info(); auto ti = log.timed_info();
filesystem::rewrite(lgr, prog, std::make_shared<dwarfs::mmap>(path), fsw); filesystem::rewrite(lgr, prog, std::make_shared<dwarfs::mmap>(path), fsw);
wg.wait(); wg_writer.wait();
ti << "filesystem rewritten"; ti << "filesystem rewritten";
} else { } else {
scanner s(lgr, wg, cfg, scanner s(lgr, wg_scanner, cfg,
entry_factory::create(no_owner, no_owner || no_time, entry_factory::create(no_owner, no_owner || no_time,
options.file_order == options.file_order ==
file_order_mode::SIMILARITY), file_order_mode::SIMILARITY),