mirror of
https://github.com/mhx/dwarfs.git
synced 2025-09-10 04:50:31 -04:00
refactor: make progress class use std::function and chrono duration
This commit is contained in:
parent
ea4f5e4b89
commit
75d6c92821
@ -26,6 +26,7 @@
|
|||||||
#include <condition_variable>
|
#include <condition_variable>
|
||||||
#include <cstddef>
|
#include <cstddef>
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
|
#include <functional>
|
||||||
#include <iosfwd>
|
#include <iosfwd>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
@ -33,8 +34,6 @@
|
|||||||
#include <thread>
|
#include <thread>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include <folly/Function.h>
|
|
||||||
|
|
||||||
#include <dwarfs/speedometer.h>
|
#include <dwarfs/speedometer.h>
|
||||||
#include <dwarfs/terminal.h>
|
#include <dwarfs/terminal.h>
|
||||||
|
|
||||||
@ -63,10 +62,15 @@ class progress {
|
|||||||
speedometer<uint64_t> speed{std::chrono::seconds(5)};
|
speedometer<uint64_t> speed{std::chrono::seconds(5)};
|
||||||
};
|
};
|
||||||
|
|
||||||
using status_function_type =
|
using update_function_type = std::function<void(progress&, bool)>;
|
||||||
folly::Function<std::string(progress const&, size_t) const>;
|
|
||||||
|
using status_function_type =
|
||||||
|
std::function<std::string(progress const&, size_t)>;
|
||||||
|
|
||||||
|
progress();
|
||||||
|
explicit progress(update_function_type func);
|
||||||
|
progress(update_function_type func, std::chrono::microseconds interval);
|
||||||
|
|
||||||
progress(folly::Function<void(progress&, bool)>&& func, unsigned interval_ms);
|
|
||||||
~progress() noexcept;
|
~progress() noexcept;
|
||||||
|
|
||||||
void set_status_function(status_function_type status_fun);
|
void set_status_function(status_function_type status_fun);
|
||||||
@ -149,7 +153,7 @@ class progress {
|
|||||||
void add_context(std::shared_ptr<context> const& ctx) const;
|
void add_context(std::shared_ptr<context> const& ctx) const;
|
||||||
|
|
||||||
mutable std::mutex running_mx_;
|
mutable std::mutex running_mx_;
|
||||||
bool running_;
|
bool running_{false};
|
||||||
mutable std::mutex mx_;
|
mutable std::mutex mx_;
|
||||||
std::condition_variable cond_;
|
std::condition_variable cond_;
|
||||||
std::shared_ptr<status_function_type> status_fun_;
|
std::shared_ptr<status_function_type> status_fun_;
|
||||||
|
@ -30,10 +30,15 @@
|
|||||||
|
|
||||||
namespace dwarfs {
|
namespace dwarfs {
|
||||||
|
|
||||||
progress::progress(folly::Function<void(progress&, bool)>&& func,
|
progress::progress() {}
|
||||||
unsigned interval_ms)
|
|
||||||
|
progress::progress(update_function_type func)
|
||||||
|
: progress(std::move(func), std::chrono::seconds(1)) {}
|
||||||
|
|
||||||
|
progress::progress(update_function_type func,
|
||||||
|
std::chrono::microseconds interval)
|
||||||
: running_(true)
|
: running_(true)
|
||||||
, thread_([this, interval_ms, func = std::move(func)]() mutable {
|
, thread_([this, interval, func = std::move(func)]() mutable {
|
||||||
folly::setThreadName("progress");
|
folly::setThreadName("progress");
|
||||||
#ifdef _WIN32
|
#ifdef _WIN32
|
||||||
::SetThreadPriority(::GetCurrentThread(), THREAD_PRIORITY_HIGHEST);
|
::SetThreadPriority(::GetCurrentThread(), THREAD_PRIORITY_HIGHEST);
|
||||||
@ -41,13 +46,14 @@ progress::progress(folly::Function<void(progress&, bool)>&& func,
|
|||||||
std::unique_lock lock(running_mx_);
|
std::unique_lock lock(running_mx_);
|
||||||
while (running_) {
|
while (running_) {
|
||||||
func(*this, false);
|
func(*this, false);
|
||||||
cond_.wait_for(lock, std::chrono::milliseconds(interval_ms));
|
cond_.wait_for(lock, interval);
|
||||||
}
|
}
|
||||||
func(*this, true);
|
func(*this, true);
|
||||||
}) {
|
}) {
|
||||||
}
|
}
|
||||||
|
|
||||||
progress::~progress() noexcept {
|
progress::~progress() noexcept {
|
||||||
|
if (running_) {
|
||||||
try {
|
try {
|
||||||
{
|
{
|
||||||
std::lock_guard lock(running_mx_);
|
std::lock_guard lock(running_mx_);
|
||||||
@ -57,6 +63,7 @@ progress::~progress() noexcept {
|
|||||||
thread_.join();
|
thread_.join();
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void progress::add_context(std::shared_ptr<context> const& ctx) const {
|
void progress::add_context(std::shared_ptr<context> const& ctx) const {
|
||||||
|
@ -369,6 +369,7 @@ void validate(boost::any& v, std::vector<std::string> const& values,
|
|||||||
|
|
||||||
int mkdwarfs_main(int argc, sys_char** argv, iolayer const& iol) {
|
int mkdwarfs_main(int argc, sys_char** argv, iolayer const& iol) {
|
||||||
using namespace folly::gen;
|
using namespace folly::gen;
|
||||||
|
using namespace std::chrono_literals;
|
||||||
|
|
||||||
const size_t num_cpu = std::max(folly::hardware_concurrency(), 1u);
|
const size_t num_cpu = std::max(folly::hardware_concurrency(), 1u);
|
||||||
static constexpr size_t const kDefaultMaxActiveBlocks{1};
|
static constexpr size_t const kDefaultMaxActiveBlocks{1};
|
||||||
@ -1062,10 +1063,10 @@ int mkdwarfs_main(int argc, sys_char** argv, iolayer const& iol) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
unsigned interval_ms =
|
auto interval =
|
||||||
pg_mode == console_writer::NONE || pg_mode == console_writer::SIMPLE
|
pg_mode == console_writer::NONE || pg_mode == console_writer::SIMPLE
|
||||||
? 2000
|
? 2000ms
|
||||||
: 200;
|
: 200ms;
|
||||||
|
|
||||||
filesystem_writer_options fswopts;
|
filesystem_writer_options fswopts;
|
||||||
fswopts.max_queue_size = mem_limit;
|
fswopts.max_queue_size = mem_limit;
|
||||||
@ -1088,7 +1089,7 @@ int mkdwarfs_main(int argc, sys_char** argv, iolayer const& iol) {
|
|||||||
|
|
||||||
LOG_PROXY(debug_logger_policy, lgr);
|
LOG_PROXY(debug_logger_policy, lgr);
|
||||||
|
|
||||||
folly::Function<void(progress&, bool)> updater;
|
progress::update_function_type updater;
|
||||||
|
|
||||||
if (options.debug_filter_function) {
|
if (options.debug_filter_function) {
|
||||||
updater = [](progress&, bool) {};
|
updater = [](progress&, bool) {};
|
||||||
@ -1096,7 +1097,7 @@ int mkdwarfs_main(int argc, sys_char** argv, iolayer const& iol) {
|
|||||||
updater = [&](progress& p, bool last) { lgr.update(p, last); };
|
updater = [&](progress& p, bool last) { lgr.update(p, last); };
|
||||||
}
|
}
|
||||||
|
|
||||||
progress prog(std::move(updater), interval_ms);
|
progress prog(std::move(updater), interval);
|
||||||
|
|
||||||
// No more streaming to iol.err after this point as this would
|
// No more streaming to iol.err after this point as this would
|
||||||
// cause a race with the progress thread.
|
// cause a race with the progress thread.
|
||||||
|
@ -1115,7 +1115,7 @@ TEST_P(rewrite, filesystem_rewrite) {
|
|||||||
|
|
||||||
worker_group wg(lgr, os, "rewriter", 2);
|
worker_group wg(lgr, os, "rewriter", 2);
|
||||||
block_compressor bc("null");
|
block_compressor bc("null");
|
||||||
progress prog([](const progress&, bool) {}, 1000);
|
progress prog;
|
||||||
std::ostringstream rewritten, idss;
|
std::ostringstream rewritten, idss;
|
||||||
|
|
||||||
auto rewrite_fs = [&](auto& fsw, auto const& mm) {
|
auto rewrite_fs = [&](auto& fsw, auto const& mm) {
|
||||||
|
@ -120,7 +120,7 @@ std::string make_filesystem(::benchmark::State const& state) {
|
|||||||
auto os = test::os_access_mock::create_test_instance();
|
auto os = test::os_access_mock::create_test_instance();
|
||||||
|
|
||||||
worker_group wg(lgr, *os, "writer", 4);
|
worker_group wg(lgr, *os, "writer", 4);
|
||||||
progress prog([](const progress&, bool) {}, 1000);
|
progress prog;
|
||||||
|
|
||||||
auto sf = std::make_shared<segmenter_factory>(lgr, prog, cfg);
|
auto sf = std::make_shared<segmenter_factory>(lgr, prog, cfg);
|
||||||
|
|
||||||
|
@ -78,7 +78,7 @@ build_dwarfs(logger& lgr, std::shared_ptr<test::os_access_mock> input,
|
|||||||
|
|
||||||
std::unique_ptr<progress> local_prog;
|
std::unique_ptr<progress> local_prog;
|
||||||
if (!prog) {
|
if (!prog) {
|
||||||
local_prog = std::make_unique<progress>([](const progress&, bool) {}, 1000);
|
local_prog = std::make_unique<progress>();
|
||||||
prog = local_prog.get();
|
prog = local_prog.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -161,7 +161,7 @@ void basic_end_to_end_test(std::string const& compressor,
|
|||||||
input->set_access_fail("/somedir/ipsum.py");
|
input->set_access_fail("/somedir/ipsum.py");
|
||||||
}
|
}
|
||||||
|
|
||||||
auto prog = progress([](const progress&, bool) {}, 1000);
|
progress prog;
|
||||||
|
|
||||||
auto scr = std::make_shared<test::script_mock>();
|
auto scr = std::make_shared<test::script_mock>();
|
||||||
|
|
||||||
@ -928,7 +928,7 @@ class filter_test
|
|||||||
debug_filter_output(oss, exclude, pe, mode);
|
debug_filter_output(oss, exclude, pe, mode);
|
||||||
};
|
};
|
||||||
|
|
||||||
progress prog([](const progress&, bool) {}, 1000);
|
progress prog;
|
||||||
worker_group wg(lgr, *input, "worker", 1);
|
worker_group wg(lgr, *input, "worker", 1);
|
||||||
auto sf = std::make_shared<segmenter_factory>(lgr, prog,
|
auto sf = std::make_shared<segmenter_factory>(lgr, prog,
|
||||||
segmenter_factory::config{});
|
segmenter_factory::config{});
|
||||||
|
@ -139,7 +139,7 @@ void run_segmenter_test(unsigned iters, unsigned granularity,
|
|||||||
|
|
||||||
for (unsigned i = 0; i < iters; ++i) {
|
for (unsigned i = 0; i < iters; ++i) {
|
||||||
dwarfs::test::test_logger lgr;
|
dwarfs::test::test_logger lgr;
|
||||||
dwarfs::progress prog([](dwarfs::progress const&, bool) {}, 1000);
|
dwarfs::progress prog;
|
||||||
auto blkmgr = std::make_shared<dwarfs::block_manager>();
|
auto blkmgr = std::make_shared<dwarfs::block_manager>();
|
||||||
|
|
||||||
std::vector<std::shared_ptr<dwarfs::block_data>> written;
|
std::vector<std::shared_ptr<dwarfs::block_data>> written;
|
||||||
|
Loading…
x
Reference in New Issue
Block a user