From 86ff4db32bfbfc05bfbbb0ada4198ebef6d3c85c Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Tue, 4 Jul 2023 02:34:35 +0200 Subject: [PATCH] Single file progress + speedometer --- include/dwarfs/console_writer.h | 3 ++ include/dwarfs/progress.h | 3 ++ include/dwarfs/speedometer.h | 65 +++++++++++++++++++++++++++++++ src/dwarfs/block_manager.cpp | 12 ++++++ src/dwarfs/console_writer.cpp | 68 ++++++++++++++++++++++++--------- 5 files changed, 133 insertions(+), 18 deletions(-) create mode 100644 include/dwarfs/speedometer.h diff --git a/include/dwarfs/console_writer.h b/include/dwarfs/console_writer.h index 8355685e..72117afe 100644 --- a/include/dwarfs/console_writer.h +++ b/include/dwarfs/console_writer.h @@ -29,6 +29,7 @@ #include #include "dwarfs/logger.h" +#include "dwarfs/speedometer.h" namespace dwarfs { @@ -66,5 +67,7 @@ class console_writer : public logger { bool const color_; bool const with_context_; bool const debug_progress_; + bool writing_{false}; + speedometer read_speed_; }; } // namespace dwarfs diff --git a/include/dwarfs/progress.h b/include/dwarfs/progress.h index 1ff8cb11..d607a311 100644 --- a/include/dwarfs/progress.h +++ b/include/dwarfs/progress.h @@ -55,6 +55,9 @@ class progress { std::string status(size_t max_len) const; std::atomic current{nullptr}; + std::atomic total_bytes_read{0}; + std::atomic current_size{0}; + std::atomic current_offset{0}; std::atomic files_found{0}; std::atomic files_scanned{0}; std::atomic dirs_found{0}; diff --git a/include/dwarfs/speedometer.h b/include/dwarfs/speedometer.h new file mode 100644 index 00000000..526ca1a9 --- /dev/null +++ b/include/dwarfs/speedometer.h @@ -0,0 +1,65 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +#pragma once + +#include +#include + +namespace dwarfs { + +template +class speedometer { + public: + speedometer(std::chrono::milliseconds window_length) : window_length_{window_length} {} + + void put(T s) { + auto now = std::chrono::steady_clock::now(); + auto old = now - window_length_; + + while (!samples_.empty() && samples_.front().first < old) { + samples_.pop_front(); + } + + samples_.emplace_back(now, s); + } + + T num_per_second() const { + if (samples_.size() < 2) { + return T(); + } + auto const& first = samples_.front(); + auto const& last = samples_.back(); + auto dt = last.first - first.first; + auto dv = last.second - first.second; + return (1000 * dv) / std::chrono::duration_cast(dt).count(); + } + + void clear() { + samples_.clear(); + } + + private: + std::deque> samples_; + std::chrono::milliseconds window_length_; +}; + +} // namespace dwarfs diff --git a/src/dwarfs/block_manager.cpp b/src/dwarfs/block_manager.cpp index 285836a9..aceb2475 100644 --- a/src/dwarfs/block_manager.cpp +++ b/src/dwarfs/block_manager.cpp @@ -608,6 +608,10 @@ void block_manager_::segment_and_add_data(inode& ino, mmif& mm, std::vector matches; const bool single_block_mode = cfg_.max_active_blocks == 1; + auto total_bytes_read_before = prog_.total_bytes_read.load(); + prog_.current_offset.store(offset); + prog_.current_size.store(size); + while (offset < size) { ++stats_.bloom_lookups; if (DWARFS_UNLIKELY(filter_.test(hasher()))) { @@ -677,6 +681,9 @@ void block_manager_::segment_and_add_data(inode& ino, mmif& mm, hasher.update(p[offset]); } + prog_.current_offset.store(offset); + prog_.total_bytes_read.store(total_bytes_read_before + offset); + next_hash_offset = written + lookback_size + blocks_.back().next_hash_distance(); } @@ -697,12 +704,17 @@ void block_manager_::segment_and_add_data(inode& ino, mmif& mm, add_data(ino, mm, written, num_to_write); written += num_to_write; next_hash_offset += window_step_; + prog_.current_offset.store(offset); + prog_.total_bytes_read.store(total_bytes_read_before + offset); } hasher.update(p[offset - window_size_], p[offset]); ++offset; } + prog_.current_offset.store(size); + prog_.total_bytes_read.store(total_bytes_read_before + size); + add_data(ino, mm, written, size - written); finish_chunk(ino); } diff --git a/src/dwarfs/console_writer.cpp b/src/dwarfs/console_writer.cpp index 3077e9ef..7c67b631 100644 --- a/src/dwarfs/console_writer.cpp +++ b/src/dwarfs/console_writer.cpp @@ -55,6 +55,27 @@ bool is_debug_progress() { return false; } +std::string progress_bar(size_t width, double frac, bool unicode) { + size_t barlen = 8 * width * frac; + size_t w = barlen / 8; + size_t c = barlen % 8; + + auto bar = unicode ? uni_bar.data() : asc_bar.data(); + std::string rv; + + for (size_t i = 0; i < width; ++i) { + if (i == (width - 1)) { + rv.append(bar[0]); + } else if (i == w) { + rv.append(bar[c]); + } else { + rv.append(i < w ? bar[7] : " "); + } + } + + return rv; +} + } // namespace console_writer::console_writer(std::ostream& os, progress_mode pg_mode, @@ -69,7 +90,8 @@ console_writer::console_writer(std::ostream& os, progress_mode pg_mode, , mode_(mode) , color_(stream_is_fancy_terminal(os)) , with_context_(with_context) - , debug_progress_(is_debug_progress()) { + , debug_progress_(is_debug_progress()) + , read_speed_{std::chrono::seconds(5)} { if (threshold > level_type::INFO) { set_policy(); } else { @@ -83,7 +105,7 @@ void console_writer::rewind() { switch (mode_) { case NORMAL: - lines = 8; + lines = 9; break; case REWRITE: lines = 4; @@ -175,11 +197,31 @@ void console_writer::update(const progress& p, bool last) { switch (mode_) { case NORMAL: + if (writing_) { + read_speed_.put(p.total_bytes_read.load()); + } else { + if (p.total_bytes_read.load() > 0) { + read_speed_.clear(); + writing_ = true; + } else { + read_speed_.put(p.similarity_bytes.load()); + } + } + if (fancy) { oss << terminal_colored(p.status(width_), termcolor::BOLD_CYAN, color_) << newline; } + { + auto cur_size = p.current_size.load(); + double cur_offs = std::min(p.current_offset.load(), cur_size); + double cur_frac = cur_size > 0 ? cur_offs / cur_size : 0.0; + + oss << progress_bar(64, cur_frac, pg_mode_ == UNICODE) + << size_with_unit(read_speed_.num_per_second()) << "/s" << newline; + } + oss << p.dirs_scanned << " dirs, " << p.symlinks_scanned << "/" << p.hardlinks << " soft/hard links, " << p.files_scanned << "/" << p.files_found << " files, " << p.specials_found << " other" @@ -244,16 +286,14 @@ void console_writer::update(const progress& p, bool last) { p.block_count > 0 ? double(p.blocks_written) / p.block_count : 0.0; double frac = mode_ == NORMAL ? (frac_fs + frac_comp) / 2.0 : frac_comp; + if (last) { + frac = 1.0; + } + if (frac > frac_) { frac_ = frac; } - size_t barlen = 8 * (width_ - 6) * frac_; - size_t w = barlen / 8; - size_t c = barlen % 8; - - auto bar = pg_mode_ == UNICODE ? uni_bar.data() : asc_bar.data(); - if (pg_mode_ == SIMPLE) { std::string tmp = fmt::format(" ==> {0:.0f}% done, {1} blocks/{2} written", 100 * frac_, @@ -269,16 +309,8 @@ void console_writer::update(const progress& p, bool last) { os_ << oss.str(); } } else { - for (size_t i = 0; i < width_ - 6; ++i) { - if (i == (width_ - 7)) { - oss << bar[0]; - } else if (i == w && !last) { - oss << bar[c]; - } else { - oss << (i < w ? bar[7] : " "); - } - } - oss << fmt::format("{:3.0f}% ", 100 * frac_) << "-\\|/"[counter_ % 4] + oss << progress_bar(width_ - 6, frac_, pg_mode_ == UNICODE) + << fmt::format("{:3.0f}% ", 100 * frac_) << "-\\|/"[counter_ % 4] << '\n'; ++counter_;