Single file progress + speedometer

include/dwarfs/console_writer.h

@@ -29,6 +29,7 @@
 #include <string>
 
 #include "dwarfs/logger.h"
+#include "dwarfs/speedometer.h"
 
 namespace dwarfs {
 
@@ -66,5 +67,7 @@ class console_writer : public logger {
   bool const color_;
   bool const with_context_;
   bool const debug_progress_;
+  bool writing_{false};
+  speedometer<uint64_t> read_speed_;
 };
 } // namespace dwarfs

include/dwarfs/progress.h

@@ -55,6 +55,9 @@ class progress {
   std::string status(size_t max_len) const;
 
   std::atomic<object const*> current{nullptr};
+  std::atomic<uint64_t> total_bytes_read{0};
+  std::atomic<size_t> current_size{0};
+  std::atomic<size_t> current_offset{0};
   std::atomic<size_t> files_found{0};
   std::atomic<size_t> files_scanned{0};
   std::atomic<size_t> dirs_found{0};

include/dwarfs/speedometer.h

@@ -0,0 +1,65 @@
+/* vim:set ts=2 sw=2 sts=2 et: */
+/**
+ * \author     Marcus Holland-Moritz (github@mhxnet.de)
+ * \copyright  Copyright (c) Marcus Holland-Moritz
+ *
+ * This file is part of dwarfs.
+ *
+ * dwarfs is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * dwarfs is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with dwarfs.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <chrono>
+#include <deque>
+
+namespace dwarfs {
+
+template <typename T>
+class speedometer {
+ public:
+  speedometer(std::chrono::milliseconds window_length) : window_length_{window_length} {}
+
+  void put(T s) {
+    auto now = std::chrono::steady_clock::now();
+    auto old = now - window_length_;
+
+    while (!samples_.empty() && samples_.front().first < old) {
+      samples_.pop_front();
+    }
+
+    samples_.emplace_back(now, s);
+  }
+
+  T num_per_second() const {
+    if (samples_.size() < 2) {
+      return T();
+    }
+    auto const& first = samples_.front();
+    auto const& last = samples_.back();
+    auto dt = last.first - first.first;
+    auto dv = last.second - first.second;
+    return (1000 * dv) / std::chrono::duration_cast<std::chrono::milliseconds>(dt).count();
+  }
+
+  void clear() {
+    samples_.clear();
+  }
+
+ private:
+  std::deque<std::pair<std::chrono::steady_clock::time_point, T>> samples_;
+  std::chrono::milliseconds window_length_;
+};
+
+} // namespace dwarfs

src/dwarfs/block_manager.cpp

@@ -608,6 +608,10 @@ void block_manager_<LoggerPolicy>::segment_and_add_data(inode& ino, mmif& mm,
   std::vector<segment_match> matches;
   const bool single_block_mode = cfg_.max_active_blocks == 1;
 
+  auto total_bytes_read_before = prog_.total_bytes_read.load();
+  prog_.current_offset.store(offset);
+  prog_.current_size.store(size);
+
   while (offset < size) {
     ++stats_.bloom_lookups;
     if (DWARFS_UNLIKELY(filter_.test(hasher()))) {
@@ -677,6 +681,9 @@ void block_manager_<LoggerPolicy>::segment_and_add_data(inode& ino, mmif& mm,
             hasher.update(p[offset]);
           }
 
+          prog_.current_offset.store(offset);
+          prog_.total_bytes_read.store(total_bytes_read_before + offset);
+
           next_hash_offset =
               written + lookback_size + blocks_.back().next_hash_distance();
         }
@@ -697,12 +704,17 @@ void block_manager_<LoggerPolicy>::segment_and_add_data(inode& ino, mmif& mm,
       add_data(ino, mm, written, num_to_write);
       written += num_to_write;
       next_hash_offset += window_step_;
+      prog_.current_offset.store(offset);
+      prog_.total_bytes_read.store(total_bytes_read_before + offset);
     }
 
     hasher.update(p[offset - window_size_], p[offset]);
     ++offset;
   }
 
+  prog_.current_offset.store(size);
+  prog_.total_bytes_read.store(total_bytes_read_before + size);
+
   add_data(ino, mm, written, size - written);
   finish_chunk(ino);
 }

src/dwarfs/console_writer.cpp

@@ -55,6 +55,27 @@ bool is_debug_progress() {
   return false;
 }
 
+std::string progress_bar(size_t width, double frac, bool unicode) {
+  size_t barlen = 8 * width * frac;
+  size_t w = barlen / 8;
+  size_t c = barlen % 8;
+
+  auto bar = unicode ? uni_bar.data() : asc_bar.data();
+  std::string rv;
+
+  for (size_t i = 0; i < width; ++i) {
+    if (i == (width - 1)) {
+      rv.append(bar[0]);
+    } else if (i == w) {
+      rv.append(bar[c]);
+    } else {
+      rv.append(i < w ? bar[7] : " ");
+    }
+  }
+
+  return rv;
+}
+
 } // namespace
 
 console_writer::console_writer(std::ostream& os, progress_mode pg_mode,
@@ -69,7 +90,8 @@ console_writer::console_writer(std::ostream& os, progress_mode pg_mode,
     , mode_(mode)
     , color_(stream_is_fancy_terminal(os))
     , with_context_(with_context)
-    , debug_progress_(is_debug_progress()) {
+    , debug_progress_(is_debug_progress())
+    , read_speed_{std::chrono::seconds(5)} {
   if (threshold > level_type::INFO) {
     set_policy<debug_logger_policy>();
   } else {
@@ -83,7 +105,7 @@ void console_writer::rewind() {
 
     switch (mode_) {
     case NORMAL:
-      lines = 8;
+      lines = 9;
       break;
     case REWRITE:
       lines = 4;
@@ -175,11 +197,31 @@ void console_writer::update(const progress& p, bool last) {
 
     switch (mode_) {
     case NORMAL:
+      if (writing_) {
+        read_speed_.put(p.total_bytes_read.load());
+      } else {
+        if (p.total_bytes_read.load() > 0) {
+          read_speed_.clear();
+          writing_ = true;
+        } else {
+          read_speed_.put(p.similarity_bytes.load());
+        }
+      }
+
       if (fancy) {
         oss << terminal_colored(p.status(width_), termcolor::BOLD_CYAN, color_)
             << newline;
       }
 
+      {
+        auto cur_size = p.current_size.load();
+        double cur_offs = std::min(p.current_offset.load(), cur_size);
+        double cur_frac = cur_size > 0 ? cur_offs / cur_size : 0.0;
+
+        oss << progress_bar(64, cur_frac, pg_mode_ == UNICODE)
+            << size_with_unit(read_speed_.num_per_second()) << "/s" << newline;
+      }
+
       oss << p.dirs_scanned << " dirs, " << p.symlinks_scanned << "/"
           << p.hardlinks << " soft/hard links, " << p.files_scanned << "/"
           << p.files_found << " files, " << p.specials_found << " other"
@@ -244,16 +286,14 @@ void console_writer::update(const progress& p, bool last) {
       p.block_count > 0 ? double(p.blocks_written) / p.block_count : 0.0;
   double frac = mode_ == NORMAL ? (frac_fs + frac_comp) / 2.0 : frac_comp;
 
+  if (last) {
+    frac = 1.0;
+  }
+
   if (frac > frac_) {
     frac_ = frac;
   }
 
-  size_t barlen = 8 * (width_ - 6) * frac_;
-  size_t w = barlen / 8;
-  size_t c = barlen % 8;
-
-  auto bar = pg_mode_ == UNICODE ? uni_bar.data() : asc_bar.data();
-
   if (pg_mode_ == SIMPLE) {
     std::string tmp =
         fmt::format(" ==> {0:.0f}% done, {1} blocks/{2} written", 100 * frac_,
@@ -269,16 +309,8 @@ void console_writer::update(const progress& p, bool last) {
       os_ << oss.str();
     }
   } else {
-    for (size_t i = 0; i < width_ - 6; ++i) {
+    oss << progress_bar(width_ - 6, frac_, pg_mode_ == UNICODE)
-      if (i == (width_ - 7)) {
+        << fmt::format("{:3.0f}% ", 100 * frac_) << "-\\|/"[counter_ % 4]
-        oss << bar[0];
-      } else if (i == w && !last) {
-        oss << bar[c];
-      } else {
-        oss << (i < w ? bar[7] : " ");
-      }
-    }
-    oss << fmt::format("{:3.0f}% ", 100 * frac_) << "-\\|/"[counter_ % 4]
         << '\n';
 
     ++counter_;