From 63c9e9a3c3b6acab3926dadaf888c43e663258d3 Mon Sep 17 00:00:00 2001
From: Marcus Holland-Moritz <github@mhxnet.de>
Date: Mon, 7 Dec 2020 22:16:31 +0100
Subject: [PATCH] Add support for nilsimsa-based inode ordering

Also refactor inode manager to take full control of ordering.
Nilsimsa ordering is really slow, but has the advantage that
inodes are added continuously while ordering. This allows us
to perform the bulk of the ordering in the background while
already running segmenting/compression.
---
 include/dwarfs/entry.h         |  10 +-
 include/dwarfs/inode.h         |   5 +
 include/dwarfs/inode_manager.h |  42 +++-
 include/dwarfs/options.h       |  10 +-
 src/dwarfs/entry.cpp           |  20 +-
 src/dwarfs/inode_manager.cpp   | 353 +++++++++++++++++++++++++++------
 src/dwarfs/options.cpp         |   3 +
 src/dwarfs/scanner.cpp         |  79 +++-----
 src/mkdwarfs.cpp               |  13 +-
 test/dwarfs.cpp                |  16 +-
 10 files changed, 393 insertions(+), 158 deletions(-)

diff --git a/include/dwarfs/entry.h b/include/dwarfs/entry.h
index 2607bfe5..ffd77935 100644
--- a/include/dwarfs/entry.h
+++ b/include/dwarfs/entry.h
@@ -109,9 +109,8 @@ class entry : public entry_interface {
 class file : public entry {
  public:
   file(const std::string& name, std::shared_ptr<entry> parent,
-       const struct ::stat& st, bool with_similarity)
-      : entry(name, parent, st)
-      , with_similarity_(with_similarity) {}
+       const struct ::stat& st)
+      : entry(name, parent, st) {}
 
   type_t type() const override;
   std::string_view hash() const;
@@ -119,14 +118,11 @@ class file : public entry {
   std::shared_ptr<inode> get_inode() const;
   void accept(entry_visitor& v, bool preorder) override;
   uint32_t inode_num() const override;
-  uint32_t similarity_hash() const { return similarity_hash_; }
   void scan(os_access& os, progress& prog) override;
 
  private:
   using hash_type = std::array<char, 20>;
 
-  uint32_t similarity_hash_{0};
-  const bool with_similarity_;
   hash_type hash_{0};
   std::shared_ptr<inode> inode_;
 };
@@ -195,7 +191,7 @@ class device : public entry {
 
 class entry_factory {
  public:
-  static std::unique_ptr<entry_factory> create(bool with_similarity = false);
+  static std::unique_ptr<entry_factory> create();
 
   virtual ~entry_factory() = default;
 
diff --git a/include/dwarfs/inode.h b/include/dwarfs/inode.h
index 8a157e39..812cbf57 100644
--- a/include/dwarfs/inode.h
+++ b/include/dwarfs/inode.h
@@ -35,15 +35,20 @@ struct chunk;
 }
 
 class file;
+class os_access;
+
+struct inode_options;
 
 class inode : public object {
  public:
   using files_vector = folly::small_vector<file*, 1>;
 
   virtual void set_files(files_vector&& fv) = 0;
+  virtual void scan(os_access& os, inode_options const& options) = 0;
   virtual void set_num(uint32_t num) = 0;
   virtual uint32_t num() const = 0;
   virtual uint32_t similarity_hash() const = 0;
+  virtual std::vector<uint64_t> const& nilsimsa_similarity_hash() const = 0;
   virtual size_t size() const = 0;
   virtual file const* any() const = 0;
   virtual files_vector const& files() const = 0;
diff --git a/include/dwarfs/inode_manager.h b/include/dwarfs/inode_manager.h
index 42e83ca6..c6351a44 100644
--- a/include/dwarfs/inode_manager.h
+++ b/include/dwarfs/inode_manager.h
@@ -25,23 +25,45 @@
 #include <functional>
 #include <memory>
 
+#include "dwarfs/options.h"
+
 namespace dwarfs {
 
 class inode;
+class logger;
 class script;
 
 class inode_manager {
  public:
-  static std::unique_ptr<inode_manager> create();
+  using inode_cb = std::function<void(std::shared_ptr<inode> const&)>;
 
-  virtual ~inode_manager() = default;
-  virtual std::shared_ptr<inode> create_inode() = 0;
-  virtual size_t count() const = 0;
-  virtual void order_inodes() = 0;
-  virtual void order_inodes(std::shared_ptr<script> scr) = 0;
-  virtual void order_inodes_by_similarity() = 0;
-  virtual void number_inodes(size_t first_no) = 0;
-  virtual void for_each_inode(
-      std::function<void(std::shared_ptr<inode> const&)> const& fn) const = 0;
+  inode_manager(logger& lgr);
+
+  std::shared_ptr<inode> create_inode() { return impl_->create_inode(); }
+
+  size_t count() const { return impl_->count(); }
+
+  void order_inodes(std::shared_ptr<script> scr, file_order_mode file_order,
+                    uint32_t first_inode, inode_cb const& fn) {
+    impl_->order_inodes(std::move(scr), file_order, first_inode, fn);
+  }
+
+  void for_each_inode(inode_cb const& fn) const { impl_->for_each_inode(fn); }
+
+  class impl {
+   public:
+    virtual ~impl() = default;
+
+    virtual std::shared_ptr<inode> create_inode() = 0;
+    virtual size_t count() const = 0;
+    virtual void
+    order_inodes(std::shared_ptr<script> scr, file_order_mode file_order,
+                 uint32_t first_inode, inode_cb const& fn) = 0;
+    virtual void for_each_inode(
+        std::function<void(std::shared_ptr<inode> const&)> const& fn) const = 0;
+  };
+
+ private:
+  std::unique_ptr<impl> impl_;
 };
 } // namespace dwarfs
diff --git a/include/dwarfs/options.h b/include/dwarfs/options.h
index f8c8ac14..d4fe01fb 100644
--- a/include/dwarfs/options.h
+++ b/include/dwarfs/options.h
@@ -46,7 +46,14 @@ struct filesystem_options {
   metadata_options metadata;
 };
 
-enum class file_order_mode { NONE, PATH, SCRIPT, SIMILARITY };
+struct inode_options {
+  bool with_similarity{false};
+  bool with_nilsimsa{false};
+
+  bool needs_scan() const { return with_similarity || with_nilsimsa; }
+};
+
+enum class file_order_mode { NONE, PATH, SCRIPT, SIMILARITY, NILSIMSA };
 
 struct scanner_options {
   file_order_mode file_order{file_order_mode::NONE};
@@ -54,6 +61,7 @@ struct scanner_options {
   std::optional<uint16_t> gid;
   std::optional<uint64_t> timestamp;
   bool remove_empty_dirs{false};
+  inode_options inode;
 };
 
 std::ostream& operator<<(std::ostream& os, file_order_mode mode);
diff --git a/src/dwarfs/entry.cpp b/src/dwarfs/entry.cpp
index 3dc873f8..b65f72b8 100644
--- a/src/dwarfs/entry.cpp
+++ b/src/dwarfs/entry.cpp
@@ -32,9 +32,10 @@
 #include "dwarfs/global_entry_data.h"
 #include "dwarfs/inode.h"
 #include "dwarfs/mmif.h"
+#include "dwarfs/nilsimsa.h"
+#include "dwarfs/options.h"
 #include "dwarfs/os_access.h"
 #include "dwarfs/progress.h"
-#include "dwarfs/similarity.h"
 
 #include "dwarfs/gen-cpp2/metadata_types.h"
 
@@ -168,10 +169,6 @@ void file::scan(os_access& os, progress& prog) {
     auto mm = os.map_file(path(), s);
     ::SHA1(mm->as<unsigned char>(), s,
            reinterpret_cast<unsigned char*>(&hash_[0]));
-
-    if (with_similarity_) {
-      similarity_hash_ = get_similarity_hash(mm->as<uint8_t>(), s);
-    }
   } else {
     ::SHA1(nullptr, 0, reinterpret_cast<unsigned char*>(&hash_[0]));
   }
@@ -290,9 +287,6 @@ uint64_t device::device_id() const { return status().st_rdev; }
 
 class entry_factory_ : public entry_factory {
  public:
-  entry_factory_(bool with_similarity)
-      : with_similarity_(with_similarity) {}
-
   std::shared_ptr<entry> create(os_access& os, const std::string& name,
                                 std::shared_ptr<entry> parent) override {
     const std::string& p = parent ? parent->path() + "/" + name : name;
@@ -302,8 +296,7 @@ class entry_factory_ : public entry_factory {
     auto mode = st.st_mode;
 
     if (S_ISREG(mode)) {
-      return std::make_shared<file>(name, std::move(parent), st,
-                                    with_similarity_);
+      return std::make_shared<file>(name, std::move(parent), st);
     } else if (S_ISDIR(mode)) {
       return std::make_shared<dir>(name, std::move(parent), st);
     } else if (S_ISLNK(mode)) {
@@ -317,12 +310,9 @@ class entry_factory_ : public entry_factory {
 
     return std::shared_ptr<entry>();
   }
-
- private:
-  const bool with_similarity_;
 };
 
-std::unique_ptr<entry_factory> entry_factory::create(bool with_similarity) {
-  return std::make_unique<entry_factory_>(with_similarity);
+std::unique_ptr<entry_factory> entry_factory::create() {
+  return std::make_unique<entry_factory_>();
 }
 } // namespace dwarfs
diff --git a/src/dwarfs/inode_manager.cpp b/src/dwarfs/inode_manager.cpp
index 984d26da..6bbd66c0 100644
--- a/src/dwarfs/inode_manager.cpp
+++ b/src/dwarfs/inode_manager.cpp
@@ -20,7 +20,9 @@
  */
 
 #include <algorithm>
+#include <cassert>
 #include <cstdint>
+#include <deque>
 #include <limits>
 #include <numeric>
 #include <stdexcept>
@@ -30,64 +32,125 @@
 #include "dwarfs/entry.h"
 #include "dwarfs/inode.h"
 #include "dwarfs/inode_manager.h"
+#include "dwarfs/logger.h"
+#include "dwarfs/mmif.h"
+#include "dwarfs/nilsimsa.h"
+#include "dwarfs/os_access.h"
 #include "dwarfs/script.h"
+#include "dwarfs/similarity.h"
 
 #include "dwarfs/gen-cpp2/metadata_types.h"
 
 namespace dwarfs {
 
-class inode_manager_ : public inode_manager {
- private:
-  class inode_ : public inode {
-   public:
-    using chunk_type = thrift::metadata::chunk;
-
-    void set_num(uint32_t num) override { num_ = num; }
-    uint32_t num() const override { return num_; }
-    uint32_t similarity_hash() const override {
-      if (files_.empty()) {
-        throw std::runtime_error("inode has no file");
-      }
-      return files_.front()->similarity_hash();
-    }
-
-    void set_files(files_vector&& fv) override {
-      if (!files_.empty()) {
-        throw std::runtime_error("files already set for inode");
-      }
-      files_ = std::move(fv);
-    }
-
-    void add_chunk(size_t block, size_t offset, size_t size) override {
-      chunk_type c;
-      c.block = block;
-      c.offset = offset;
-      c.size = size;
-      chunks_.push_back(c);
-    }
-
-    size_t size() const override { return any()->size(); }
-
-    files_vector const& files() const override { return files_; }
-
-    file const* any() const override {
-      if (files_.empty()) {
-        throw std::runtime_error("inode has no file");
-      }
-      return files_.front();
-    }
-
-    void append_chunks_to(std::vector<chunk_type>& vec) const override {
-      vec.insert(vec.end(), chunks_.begin(), chunks_.end());
-    }
-
-   private:
-    uint32_t num_{std::numeric_limits<uint32_t>::max()};
-    files_vector files_;
-    std::vector<chunk_type> chunks_;
-  };
+namespace {
 
+class inode_ : public inode {
  public:
+  using chunk_type = thrift::metadata::chunk;
+
+  void set_num(uint32_t num) override { num_ = num; }
+
+  uint32_t num() const override { return num_; }
+
+  uint32_t similarity_hash() const override {
+    if (files_.empty()) {
+      throw std::runtime_error("inode has no file");
+    }
+    return similarity_hash_;
+  }
+
+  std::vector<uint64_t> const& nilsimsa_similarity_hash() const override {
+    if (files_.empty()) {
+      throw std::runtime_error("inode has no file");
+    }
+    return nilsimsa_similarity_hash_;
+  }
+
+  void set_files(files_vector&& fv) override {
+    if (!files_.empty()) {
+      throw std::runtime_error("files already set for inode");
+    }
+
+    files_ = std::move(fv);
+  }
+
+  void scan(os_access& os, inode_options const& opts) override {
+    if (opts.needs_scan()) {
+      auto file = files_.front();
+      auto size = file->size();
+
+      if (size > 0) {
+        auto mm = os.map_file(file->path(), size);
+        auto data = mm->as<uint8_t>();
+
+        if (opts.with_similarity) {
+          similarity_hash_ = get_similarity_hash(data, size);
+        }
+
+        if (opts.with_nilsimsa) {
+          nilsimsa_similarity_hash_ = nilsimsa_compute_hash(data, size);
+        }
+      }
+    }
+  }
+
+  void add_chunk(size_t block, size_t offset, size_t size) override {
+    chunk_type c;
+    c.block = block;
+    c.offset = offset;
+    c.size = size;
+    chunks_.push_back(c);
+  }
+
+  size_t size() const override { return any()->size(); }
+
+  files_vector const& files() const override { return files_; }
+
+  file const* any() const override {
+    if (files_.empty()) {
+      throw std::runtime_error("inode has no file");
+    }
+    return files_.front();
+  }
+
+  void append_chunks_to(std::vector<chunk_type>& vec) const override {
+    vec.insert(vec.end(), chunks_.begin(), chunks_.end());
+  }
+
+ private:
+  uint32_t num_{std::numeric_limits<uint32_t>::max()};
+  uint32_t similarity_hash_{0};
+  files_vector files_;
+  std::vector<chunk_type> chunks_;
+  std::vector<uint64_t> nilsimsa_similarity_hash_;
+};
+
+class nilsimsa_cache_entry {
+ public:
+  nilsimsa_cache_entry(std::shared_ptr<inode> i)
+      : size(i->size())
+      , hash(i->nilsimsa_similarity_hash().data())
+      , path(i->any()->path())
+      , ino(std::move(i)) {
+    assert(hash);
+  }
+
+  int similarity{0};
+  uint64_t const size;
+  uint64_t const* const hash;
+  std::string const path;
+  std::shared_ptr<inode> ino;
+};
+
+} // namespace
+
+template <typename LoggerPolicy>
+class inode_manager_ : public inode_manager::impl {
+ public:
+  inode_manager_(logger& lgr)
+      : log_(lgr) {}
+
   std::shared_ptr<inode> create_inode() override {
     auto ino = std::make_shared<inode_>();
     inodes_.push_back(ino);
@@ -96,11 +159,59 @@ class inode_manager_ : public inode_manager {
 
   size_t count() const override { return inodes_.size(); }
 
-  void order_inodes(std::shared_ptr<script> scr) override {
-    scr->order(inodes_);
+  void order_inodes(std::shared_ptr<script> scr, file_order_mode file_order,
+                    uint32_t first_inode,
+                    inode_manager::inode_cb const& fn) override {
+    switch (file_order) {
+    case file_order_mode::NONE:
+      log_.info() << "keeping inode order";
+      break;
+
+    case file_order_mode::PATH: {
+      log_.info() << "ordering " << count() << " inodes by path name...";
+      auto ti = log_.timed_info();
+      order_inodes_by_path();
+      ti << count() << " inodes ordered";
+      break;
+    }
+
+    case file_order_mode::SCRIPT: {
+      if (!scr->has_order()) {
+        throw std::runtime_error("script cannot order inodes");
+      }
+      log_.info() << "ordering " << count() << " inodes using script...";
+      auto ti = log_.timed_info();
+      scr->order(inodes_);
+      ti << count() << " inodes ordered";
+      break;
+    }
+
+    case file_order_mode::SIMILARITY: {
+      log_.info() << "ordering " << count() << " inodes by similarity...";
+      auto ti = log_.timed_info();
+      order_inodes_by_similarity();
+      ti << count() << " inodes ordered";
+      break;
+    }
+
+    case file_order_mode::NILSIMSA: {
+      log_.info() << "ordering " << count()
+                  << " inodes using nilsimsa similarity...";
+      auto ti = log_.timed_info();
+      order_inodes_by_nilsimsa(fn, first_inode);
+      ti << count() << " inodes ordered";
+      break;
+    }
+    }
+
+    if (file_order != file_order_mode::NILSIMSA) {
+      log_.info() << "assigning file inodes...";
+      number_inodes(first_inode);
+      for_each_inode(fn);
+    }
   }
 
-  void order_inodes() override {
+  void order_inodes_by_path() {
     std::vector<std::string> paths;
     std::vector<size_t> index(inodes_.size());
 
@@ -125,7 +236,7 @@ class inode_manager_ : public inode_manager {
     inodes_.swap(tmp);
   }
 
-  void order_inodes_by_similarity() override {
+  void order_inodes_by_similarity() {
     std::sort(
         inodes_.begin(), inodes_.end(),
         [](const std::shared_ptr<inode>& a, const std::shared_ptr<inode>& b) {
@@ -138,7 +249,131 @@ class inode_manager_ : public inode_manager {
         });
   }
 
-  void number_inodes(size_t first_no) override {
+  void order_inodes_by_nilsimsa(inode_manager::inode_cb const& fn,
+                                uint32_t inode_no) {
+    auto finalize_inode = [&](auto& ino) {
+      ino->set_num(inode_no++);
+      fn(ino);
+    };
+
+    auto count = inodes_.size();
+
+    // skip all empty inodes (this is at most one)
+    auto beg = std::partition(inodes_.begin(), inodes_.end(),
+                              [](auto const& p) { return p->size() == 0; });
+
+    for (auto it = inodes_.begin(); it != beg; ++it) {
+      finalize_inode(*it);
+    }
+
+    // find the largest inode
+    std::nth_element(beg, beg, inodes_.end(), [](auto const& a, auto const& b) {
+      return (a->size() > b->size() ||
+              (a->size() == b->size() && a->any()->path() < b->any()->path()));
+    });
+
+    finalize_inode(*beg);
+
+    // build a cache for the remaining inodes
+    std::vector<nilsimsa_cache_entry> cache;
+    std::deque<uint32_t> index;
+    index.resize(std::distance(beg + 1, inodes_.end()));
+    std::iota(index.begin(), index.end(), 0);
+    cache.reserve(index.size());
+
+    for (auto it = beg + 1; it != inodes_.end(); ++it) {
+      cache.emplace_back(std::move(*it));
+    }
+
+    assert(index.size() == cache.size());
+
+    // and temporarily remove from the original array
+    inodes_.erase(beg + 1, inodes_.end());
+
+    while (!index.empty()) {
+      // compare reference inode with all remaining inodes
+      auto* ref_hash = inodes_.back()->nilsimsa_similarity_hash().data();
+      for (auto& d : cache) {
+        d.similarity = dwarfs::nilsimsa_similarity(ref_hash, d.hash);
+      }
+
+      auto cmp = [&cache](uint32_t a, uint32_t b) {
+        auto& da = cache[a];
+        auto& db = cache[b];
+        return da.similarity > db.similarity ||
+               (da.similarity == db.similarity &&
+                (da.size > db.size ||
+                 (da.size == db.size && da.path < db.path)));
+      };
+
+      size_t depth = 0;
+      size_t depth_thresh;
+      const int sim_thresh_depth = 16;
+      const int sim_thresh = 0;
+      const size_t max_depth = 2000;
+      const size_t depth_step = 500;
+
+      if (index.size() > max_depth) {
+        while (depth < max_depth && depth + depth_step < index.size()) {
+          std::partial_sort(index.begin() + depth,
+                            index.begin() + depth + depth_step, index.end(),
+                            cmp);
+          depth += depth_step;
+          if (cache[index[0]].similarity - cache[index[depth - 1]].similarity >
+              sim_thresh_depth) {
+            do {
+              --depth;
+            } while (cache[index[0]].similarity -
+                         cache[index[depth - 1]].similarity >
+                     sim_thresh_depth);
+            break;
+          }
+        }
+        depth_thresh = depth / 2;
+      } else {
+        std::sort(index.begin(), index.end(), cmp);
+        depth = index.size();
+        depth_thresh = 0;
+      }
+
+      auto sim = cache[index.front()].similarity;
+
+      while (!index.empty() && depth > depth_thresh &&
+             sim - cache[index.front()].similarity <= sim_thresh) {
+        inodes_.push_back(std::move(cache[index.front()].ino));
+        finalize_inode(inodes_.back());
+        index.pop_front();
+        --depth;
+      }
+
+      while (depth > depth_thresh) {
+        ref_hash = inodes_.back()->nilsimsa_similarity_hash().data();
+        for (size_t i = 0; i < depth; ++i) {
+          cache[index[i]].similarity =
+              dwarfs::nilsimsa_similarity(ref_hash, cache[index[i]].hash);
+        }
+
+        std::partial_sort(index.begin(), index.begin() + (depth - depth_thresh),
+                          index.begin() + depth, cmp);
+
+        sim = cache[index.front()].similarity;
+
+        while (!index.empty() && depth > depth_thresh &&
+               sim - cache[index.front()].similarity <= sim_thresh) {
+          inodes_.push_back(std::move(cache[index.front()].ino));
+          finalize_inode(inodes_.back());
+          index.pop_front();
+          --depth;
+        }
+      }
+    }
+
+    if (count != inodes_.size()) {
+      throw std::runtime_error("internal error: nilsimsa ordering failed");
+    }
+  }
+
+  void number_inodes(size_t first_no) {
     for (auto& i : inodes_) {
       i->set_num(first_no++);
     }
@@ -154,9 +389,11 @@ class inode_manager_ : public inode_manager {
 
  private:
   std::vector<std::shared_ptr<inode>> inodes_;
+  log_proxy<LoggerPolicy> log_;
 };
 
-std::unique_ptr<inode_manager> inode_manager::create() {
-  return std::make_unique<inode_manager_>();
-}
+inode_manager::inode_manager(logger& lgr)
+    : impl_(make_unique_logging_object<impl, inode_manager_, logger_policies>(
+          lgr)) {}
+
 } // namespace dwarfs
diff --git a/src/dwarfs/options.cpp b/src/dwarfs/options.cpp
index 887c73f4..f382f686 100644
--- a/src/dwarfs/options.cpp
+++ b/src/dwarfs/options.cpp
@@ -44,6 +44,9 @@ std::ostream& operator<<(std::ostream& os, file_order_mode mode) {
   case file_order_mode::SIMILARITY:
     modestr = "similarity";
     break;
+  case file_order_mode::NILSIMSA:
+    modestr = "nilsimsa";
+    break;
   default:
     break;
   }
diff --git a/src/dwarfs/scanner.cpp b/src/dwarfs/scanner.cpp
index b2dda100..36d6443c 100644
--- a/src/dwarfs/scanner.cpp
+++ b/src/dwarfs/scanner.cpp
@@ -37,6 +37,8 @@
 
 #include <folly/ExceptionString.h>
 
+#include <fmt/format.h>
+
 #include "dwarfs/entry.h"
 #include "dwarfs/filesystem_writer.h"
 #include "dwarfs/global_entry_data.h"
@@ -93,7 +95,8 @@ class file_deduplication_visitor : public visitor_base {
  public:
   void visit(file* p) override { hash_[p->hash()].push_back(p); }
 
-  void deduplicate_files(inode_manager& im, progress& prog) {
+  void deduplicate_files(worker_group& wg, os_access& os, inode_manager& im,
+                         inode_options const& ino_opts, progress& prog) {
     for (auto& p : hash_) {
       auto& files = p.second;
 
@@ -115,6 +118,10 @@ class file_deduplication_visitor : public visitor_base {
       }
 
       inode->set_files(std::move(files));
+
+      if (ino_opts.needs_scan()) {
+        wg.add_job([&, inode] { inode->scan(os, ino_opts); });
+      }
     }
   }
 
@@ -278,7 +285,6 @@ class scanner_ : public scanner::impl {
 
  private:
   std::shared_ptr<entry> scan_tree(const std::string& path, progress& prog);
-  void order_files(inode_manager& im);
 
   const block_manager::config& cfg_;
   const scanner_options& options_;
@@ -409,42 +415,6 @@ scanner_<LoggerPolicy>::scan_tree(const std::string& path, progress& prog) {
   return root;
 }
 
-template <typename LoggerPolicy>
-void scanner_<LoggerPolicy>::order_files(inode_manager& im) {
-  switch (options_.file_order) {
-  case file_order_mode::NONE:
-    log_.info() << "keeping inode order";
-    break;
-
-  case file_order_mode::PATH: {
-    log_.info() << "ordering " << im.count() << " inodes by path name...";
-    auto ti = log_.timed_info();
-    im.order_inodes();
-    ti << im.count() << " inodes ordered";
-    break;
-  }
-
-  case file_order_mode::SCRIPT: {
-    if (!script_->has_order()) {
-      throw std::runtime_error("script cannot order inodes");
-    }
-    log_.info() << "ordering " << im.count() << " inodes using script...";
-    auto ti = log_.timed_info();
-    im.order_inodes(script_);
-    ti << im.count() << " inodes ordered";
-    break;
-  }
-
-  case file_order_mode::SIMILARITY: {
-    log_.info() << "ordering " << im.count() << " inodes by similarity...";
-    auto ti = log_.timed_info();
-    im.order_inodes_by_similarity();
-    ti << im.count() << " inodes ordered";
-    break;
-  }
-  }
-}
-
 template <typename LoggerPolicy>
 void scanner_<LoggerPolicy>::scan(filesystem_writer& fsw,
                                   const std::string& path, progress& prog) {
@@ -479,22 +449,22 @@ void scanner_<LoggerPolicy>::scan(filesystem_writer& fsw,
 
   log_.info() << "finding duplicate files...";
 
-  auto im = inode_manager::create();
+  inode_manager im(lgr_);
 
   file_deduplication_visitor fdv;
   root->accept(fdv);
 
-  fdv.deduplicate_files(*im, prog);
+  fdv.deduplicate_files(wg_, *os_, im, options_.inode, prog);
 
   log_.info() << "saved " << size_with_unit(prog.saved_by_deduplication)
               << " / " << size_with_unit(prog.original_size) << " in "
               << prog.duplicate_files << "/" << prog.files_found
               << " duplicate files";
 
-  order_files(*im);
-
-  log_.info() << "assigning file inodes...";
-  im->number_inodes(first_file_inode);
+  if (options_.inode.needs_scan()) {
+    log_.info() << "waiting for inode scanners...";
+    wg_.wait();
+  }
 
   global_entry_data ge_data(options_);
   thrift::metadata::metadata mv2;
@@ -502,7 +472,7 @@ void scanner_<LoggerPolicy>::scan(filesystem_writer& fsw,
   mv2.link_index.resize(first_file_inode - first_link_inode);
 
   log_.info() << "assigning device inodes...";
-  uint32_t first_device_inode = first_file_inode + im->count();
+  uint32_t first_device_inode = first_file_inode + im.count();
   device_set_inode_visitor devsiv(first_device_inode);
   root->accept(devsiv);
   mv2.devices_ref() = std::move(devsiv.device_ids());
@@ -534,11 +504,12 @@ void scanner_<LoggerPolicy>::scan(filesystem_writer& fsw,
   log_.info() << "building blocks...";
   block_manager bm(lgr_, prog, cfg_, os_, fsw);
 
-  im->for_each_inode([&](std::shared_ptr<inode> const& ino) {
-    prog.current.store(ino.get());
-    bm.add_inode(ino);
-    prog.inodes_written++;
-  });
+  im.order_inodes(script_, options_.file_order, first_file_inode,
+                  [&](std::shared_ptr<inode> const& ino) {
+                    prog.current.store(ino.get());
+                    bm.add_inode(ino);
+                    prog.inodes_written++;
+                  });
 
   log_.info() << "waiting for block compression to finish...";
 
@@ -567,19 +538,19 @@ void scanner_<LoggerPolicy>::scan(filesystem_writer& fsw,
   root->set_name(std::string());
 
   log_.info() << "saving chunks...";
-  mv2.chunk_index.resize(im->count() + 1);
+  mv2.chunk_index.resize(im.count() + 1);
 
   // TODO: we should be able to start this once all blocks have been
   //       submitted for compression
-  im->for_each_inode([&](std::shared_ptr<inode> const& ino) {
+  im.for_each_inode([&](std::shared_ptr<inode> const& ino) {
     mv2.chunk_index.at(ino->num() - first_file_inode) = mv2.chunks.size();
     ino->append_chunks_to(mv2.chunks);
   });
 
   // insert dummy inode to help determine number of chunks per inode
-  mv2.chunk_index.at(im->count()) = mv2.chunks.size();
+  mv2.chunk_index.at(im.count()) = mv2.chunks.size();
 
-  log_.debug() << "total number of file inodes: " << im->count();
+  log_.debug() << "total number of file inodes: " << im.count();
   log_.debug() << "total number of chunks: " << mv2.chunks.size();
 
   log_.info() << "saving directories...";
diff --git a/src/mkdwarfs.cpp b/src/mkdwarfs.cpp
index 778f04f8..3c421076 100644
--- a/src/mkdwarfs.cpp
+++ b/src/mkdwarfs.cpp
@@ -95,7 +95,8 @@ const std::map<std::string, file_order_mode> order_choices{
 #ifdef DWARFS_HAVE_PYTHON
     {"script", file_order_mode::SCRIPT},
 #endif
-    {"similarity", file_order_mode::SIMILARITY}};
+    {"similarity", file_order_mode::SIMILARITY},
+    {"nilsimsa", file_order_mode::NILSIMSA}};
 
 } // namespace
 
@@ -555,10 +556,12 @@ int mkdwarfs(int argc, char** argv) {
     wg_writer.wait();
     ti << "filesystem rewritten";
   } else {
-    scanner s(lgr, wg_scanner, cfg,
-              entry_factory::create(force_similarity ||
-                                    options.file_order ==
-                                        file_order_mode::SIMILARITY),
+    options.inode.with_similarity =
+        force_similarity || options.file_order == file_order_mode::SIMILARITY;
+    options.inode.with_nilsimsa =
+        options.file_order == file_order_mode::NILSIMSA;
+
+    scanner s(lgr, wg_scanner, cfg, entry_factory::create(),
               std::make_shared<os_access_posix>(), std::move(script), options);
 
     {
diff --git a/test/dwarfs.cpp b/test/dwarfs.cpp
index 014d8713..eb54b962 100644
--- a/test/dwarfs.cpp
+++ b/test/dwarfs.cpp
@@ -199,6 +199,8 @@ void basic_end_to_end_test(const std::string& compressor,
   cfg.block_size_bits = block_size_bits;
 
   options.file_order = file_order;
+  options.inode.with_similarity = file_order == file_order_mode::SIMILARITY;
+  options.inode.with_nilsimsa = file_order == file_order_mode::NILSIMSA;
 
   // force multithreading
   worker_group wg("writer", 4);
@@ -207,8 +209,7 @@ void basic_end_to_end_test(const std::string& compressor,
   stream_logger lgr(logss); // TODO: mock
   lgr.set_policy<prod_logger_policy>();
 
-  scanner s(lgr, wg, cfg,
-            entry_factory::create(file_order == file_order_mode::SIMILARITY),
+  scanner s(lgr, wg, cfg, entry_factory::create(),
             std::make_shared<test::os_access_mock>(),
             std::make_shared<test::script_mock>(), options);
 
@@ -323,9 +324,8 @@ TEST_P(basic, end_to_end) {
 
 INSTANTIATE_TEST_SUITE_P(
     dwarfs, basic,
-    ::testing::Combine(::testing::ValuesIn(compressions),
-                       ::testing::Values(12, 15, 20, 28),
-                       ::testing::Values(file_order_mode::NONE,
-                                         file_order_mode::PATH,
-                                         file_order_mode::SCRIPT,
-                                         file_order_mode::SIMILARITY)));
+    ::testing::Combine(
+        ::testing::ValuesIn(compressions), ::testing::Values(12, 15, 20, 28),
+        ::testing::Values(file_order_mode::NONE, file_order_mode::PATH,
+                          file_order_mode::SCRIPT, file_order_mode::NILSIMSA,
+                          file_order_mode::SIMILARITY)));