Integrate categorizers into inode manager

CMakeLists.txt

@@ -359,6 +359,7 @@ list(
   src/dwarfs/builtin_script.cpp
   src/dwarfs/cached_block.cpp
   src/dwarfs/categorizer.cpp
+  src/dwarfs/category_parser.cpp
   src/dwarfs/checksum.cpp
   src/dwarfs/chmod_transformer.cpp
   src/dwarfs/console_writer.cpp
@@ -371,6 +372,7 @@ list(
   src/dwarfs/filesystem_extractor.cpp
   src/dwarfs/filesystem_v2.cpp
   src/dwarfs/filesystem_writer.cpp
+  src/dwarfs/fragment_order_parser.cpp
   src/dwarfs/fstypes.cpp
   src/dwarfs/fs_section.cpp
   src/dwarfs/global_entry_data.cpp

include/dwarfs/categorizer.h

@@ -27,6 +27,7 @@
 #include <limits>
 #include <map>
 #include <memory>
+#include <optional>
 #include <span>
 #include <string_view>
 
@@ -124,6 +125,8 @@ class categorizer_manager {
  public:
   categorizer_manager(logger& lgr);
 
+  static fragment_category default_category();
+
   void add(std::shared_ptr<categorizer const> c) { impl_->add(std::move(c)); }
 
   categorizer_job job(std::filesystem::path const& path) const {
@@ -134,6 +137,11 @@ class categorizer_manager {
     return impl_->category_name(c);
   }
 
+  std::optional<fragment_category::value_type>
+  category_value(std::string_view name) const {
+    return impl_->category_value(name);
+  }
+
   folly::dynamic category_metadata(fragment_category c) const {
     return impl_->category_metadata(c);
   }
@@ -146,6 +154,8 @@ class categorizer_manager {
     virtual categorizer_job job(std::filesystem::path const& path) const = 0;
     virtual std::string_view
     category_name(fragment_category::value_type c) const = 0;
+    virtual std::optional<fragment_category::value_type>
+    category_value(std::string_view name) const = 0;
     virtual folly::dynamic category_metadata(fragment_category c) const = 0;
   };
 

include/dwarfs/category_parser.h

@@ -0,0 +1,43 @@
+/* vim:set ts=2 sw=2 sts=2 et: */
+/**
+ * \author     Marcus Holland-Moritz (github@mhxnet.de)
+ * \copyright  Copyright (c) Marcus Holland-Moritz
+ *
+ * This file is part of dwarfs.
+ *
+ * dwarfs is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * dwarfs is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with dwarfs.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <memory>
+#include <vector>
+
+#include "dwarfs/fragment_category.h"
+
+namespace dwarfs {
+
+class categorizer_manager;
+
+class category_parser {
+ public:
+  category_parser(std::shared_ptr<categorizer_manager> catmgr);
+
+  std::vector<fragment_category::value_type> parse(std::string_view arg) const;
+
+ private:
+  std::shared_ptr<categorizer_manager> catmgr_;
+};
+
+} // namespace dwarfs

include/dwarfs/contextual_option.h

@@ -0,0 +1,158 @@
+/* vim:set ts=2 sw=2 sts=2 et: */
+/**
+ * \author     Marcus Holland-Moritz (github@mhxnet.de)
+ * \copyright  Copyright (c) Marcus Holland-Moritz
+ *
+ * This file is part of dwarfs.
+ *
+ * dwarfs is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * dwarfs is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with dwarfs.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <optional>
+#include <span>
+#include <stdexcept>
+#include <type_traits>
+#include <unordered_map>
+
+#include <fmt/format.h>
+
+namespace dwarfs {
+
+template <typename Policy>
+class contextual_option {
+ public:
+  using policy_type = Policy;
+  using context_argument_type = typename policy_type::ContextArgumentType;
+  using context_type = typename policy_type::ContextType;
+  using option_type = typename policy_type::OptionType;
+
+  contextual_option() = default;
+  explicit contextual_option(option_type const& def)
+      : default_{def} {}
+
+  void set_default(option_type const& val) { default_ = val; }
+
+  void add_contextual(context_type const& ctx, option_type const& val) {
+    contextual_[ctx] = val;
+  }
+
+  std::optional<option_type>
+  get_optional(context_argument_type const& arg) const {
+    if constexpr (std::is_same_v<context_type, context_argument_type>) {
+      return get_optional_impl(arg);
+    } else {
+      return get_optional_impl(policy_type::context_from_arg(arg));
+    }
+  }
+
+  option_type get(context_argument_type const& arg) const {
+    if constexpr (std::is_same_v<context_type, context_argument_type>) {
+      return get_impl(arg);
+    } else {
+      return get_impl(policy_type::context_from_arg(arg));
+    }
+  }
+
+  std::optional<option_type> get_optional() const { return default_; }
+
+  option_type get() const { return default_.value(); }
+
+  template <typename T>
+  bool any_is(T&& pred) const {
+    for (auto e : contextual_) {
+      if (pred(e.second)) {
+        return true;
+      }
+    }
+    return default_ && pred(*default_);
+  }
+
+ private:
+  std::optional<option_type> get_optional_impl(context_type const& ctx) const {
+    if (auto it = contextual_.find(ctx); it != contextual_.end()) {
+      return it->second;
+    }
+    return default_;
+  }
+
+  option_type get_impl(context_type const& ctx) const {
+    if (auto it = contextual_.find(ctx); it != contextual_.end()) {
+      return it->second;
+    }
+    return default_.value();
+  }
+
+  std::optional<option_type> default_;
+  std::unordered_map<context_type, option_type> contextual_;
+};
+
+template <typename OptionType, typename ContextParser, typename OptionParser>
+class contextual_option_parser {
+ public:
+  using option_type = OptionType;
+  using policy_type = typename option_type::policy_type;
+
+  contextual_option_parser(OptionType& opt, ContextParser const& cp,
+                           OptionParser const& op)
+      : opt_{opt}
+      , cp_{cp}
+      , op_{op} {}
+
+  void parse(std::string_view arg) const {
+    try {
+      auto pos = arg.find("::");
+
+      if (pos == arg.npos) {
+        opt_.set_default(op_.parse(arg));
+      } else {
+        auto ctx = arg.substr(0, pos);
+        auto val = op_.parse(arg.substr(pos + 2));
+        if constexpr (std::is_same_v<
+                          std::invoke_result_t<decltype(&ContextParser::parse),
+                                               ContextParser, decltype(ctx)>,
+                          typename option_type::context_type>) {
+          opt_.add_contextual(cp_.parse(ctx), val);
+        } else {
+          for (auto c : cp_.parse(ctx)) {
+            opt_.add_contextual(c, val);
+          }
+        }
+      }
+    } catch (std::exception const& e) {
+      throw std::runtime_error(
+          fmt::format("failed to parse: {} ({})", arg, e.what()));
+    }
+  }
+
+  void parse(std::span<std::string const> list) const {
+    for (auto const& arg : list) {
+      parse(arg);
+    }
+  }
+
+  void parse(std::span<std::string_view const> list) const {
+    for (auto const& arg : list) {
+      parse(arg);
+    }
+  }
+
+ private:
+  OptionType& opt_;
+  ContextParser const& cp_;
+  OptionParser const& op_;
+};
+
+} // namespace dwarfs

include/dwarfs/file_scanner.h

@@ -40,7 +40,6 @@ namespace detail {
 class file_scanner {
  public:
   file_scanner(worker_group& wg, os_access& os, inode_manager& im,
-               inode_options const& ino_opts,
                std::optional<std::string> const& hash_algo, progress& prog);
 
   void scan(file* p) { impl_->scan(p); }

include/dwarfs/fragment_category.h

@@ -25,6 +25,8 @@
 #include <cstdint>
 #include <limits>
 
+#include <folly/hash/Hash.h>
+
 namespace dwarfs {
 
 class fragment_category {
@@ -88,9 +90,26 @@ class fragment_category {
     return subcategory_;
   }
 
+  auto operator<=>(fragment_category const&) const = default;
+
+  size_t hash() const {
+    return folly::hash::hash_combine(value_, subcategory_);
+  }
+
  private:
   value_type value_{uninitialized};
   value_type subcategory_{uninitialized};
 };
 
 } // namespace dwarfs
+
+namespace std {
+
+template <>
+struct hash<dwarfs::fragment_category> {
+  std::size_t operator()(dwarfs::fragment_category const& k) const {
+    return k.hash();
+  }
+};
+
+} // namespace std

include/dwarfs/fragment_order_parser.h

@@ -0,0 +1,37 @@
+/* vim:set ts=2 sw=2 sts=2 et: */
+/**
+ * \author     Marcus Holland-Moritz (github@mhxnet.de)
+ * \copyright  Copyright (c) Marcus Holland-Moritz
+ *
+ * This file is part of dwarfs.
+ *
+ * dwarfs is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * dwarfs is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with dwarfs.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <string_view>
+
+#include "dwarfs/options.h"
+
+namespace dwarfs {
+
+struct fragment_order_parser {
+ public:
+  static std::string choices();
+
+  file_order_options parse(std::string_view arg) const;
+};
+
+} // namespace dwarfs

include/dwarfs/inode.h

@@ -47,7 +47,6 @@ class inode : public object {
   using files_vector = folly::small_vector<file*, 1>;
 
   virtual void set_files(files_vector&& fv) = 0;
-  virtual void set_similarity_valid(inode_options const& opts) = 0;
   virtual void scan(mmif* mm, inode_options const& options) = 0;
   virtual void set_num(uint32_t num) = 0;
   virtual uint32_t num() const = 0;

include/dwarfs/inode_fragments.h

@@ -65,10 +65,17 @@ class inode_fragments {
 
   std::span<single_inode_fragment const> span() const { return fragments_; }
 
+  size_t size() const { return fragments_.size(); }
+
   bool empty() const { return fragments_.empty(); }
 
   void clear() { fragments_.clear(); }
 
+  fragment_category get_single_category() const {
+    assert(fragments_.size() == 1);
+    return fragments_.at(0).category();
+  }
+
   explicit operator bool() const { return !empty(); }
 
   std::ostream&

include/dwarfs/inode_manager.h

@@ -32,27 +32,29 @@
 
 namespace dwarfs {
 
+class file;
 class inode;
 class logger;
+class os_access;
 class progress;
 class script;
+class worker_group;
 
-struct file_order_options;
+struct inode_options;
 
 class inode_manager {
  public:
   using inode_cb = std::function<void(std::shared_ptr<inode> const&)>;
   using order_cb = std::function<int64_t(std::shared_ptr<inode> const&)>;
 
-  inode_manager(logger& lgr, progress& prog);
+  inode_manager(logger& lgr, progress& prog, inode_options const& opts);
 
   std::shared_ptr<inode> create_inode() { return impl_->create_inode(); }
 
   size_t count() const { return impl_->count(); }
 
-  void order_inodes(std::shared_ptr<script> scr,
+  void order_inodes(std::shared_ptr<script> scr, order_cb const& fn) {
-                    file_order_options const& file_order, order_cb const& fn) {
+    impl_->order_inodes(std::move(scr), fn);
-    impl_->order_inodes(std::move(scr), file_order, fn);
   }
 
   void for_each_inode_in_order(inode_cb const& fn) const {
@@ -64,6 +66,11 @@ class inode_manager {
     return impl_->category_counts();
   }
 
+  void scan_background(worker_group& wg, os_access& os,
+                       std::shared_ptr<inode> ino, file const* p) const {
+    impl_->scan_background(wg, os, std::move(ino), p);
+  }
+
   class impl {
    public:
     virtual ~impl() = default;
@@ -71,12 +78,14 @@ class inode_manager {
     virtual std::shared_ptr<inode> create_inode() = 0;
     virtual size_t count() const = 0;
     virtual void
-    order_inodes(std::shared_ptr<script> scr,
+    order_inodes(std::shared_ptr<script> scr, order_cb const& fn) = 0;
-                 file_order_options const& file_order, order_cb const& fn) = 0;
     virtual void for_each_inode_in_order(
         std::function<void(std::shared_ptr<inode> const&)> const& fn) const = 0;
     virtual std::vector<std::pair<fragment_category::value_type, size_t>>
     category_counts() const = 0;
+    virtual void
+    scan_background(worker_group& wg, os_access& os, std::shared_ptr<inode> ino,
+                    file const* p) const = 0;
   };
 
  private:

include/dwarfs/nilsimsa.h

@@ -24,6 +24,7 @@
 #include <array>
 #include <cstdint>
 #include <memory>
+#include <span>
 #include <type_traits>
 
 #include <folly/lang/Bits.h>
@@ -60,6 +61,10 @@ class nilsimsa {
   static int
   similarity(uint64_t const* a, uint64_t const* b);
 
+  void operator()(std::span<uint8_t const> data) {
+    update(data.data(), data.size());
+  }
+
  private:
   class impl;
 

include/dwarfs/options.h

@@ -28,7 +28,9 @@
 #include <memory>
 #include <optional>
 
+#include "dwarfs/contextual_option.h"
 #include "dwarfs/file_stat.h"
+#include "dwarfs/fragment_category.h"
 #include "dwarfs/types.h"
 
 namespace dwarfs {
@@ -36,6 +38,25 @@ namespace dwarfs {
 class categorizer_manager;
 class entry;
 
+namespace detail {
+
+template <typename T>
+struct categorized_option_policy {
+  using ContextArgumentType = fragment_category;
+  using ContextType = fragment_category::value_type;
+  using OptionType = T;
+
+  static ContextType context_from_arg(ContextArgumentType const& arg) {
+    return arg.value();
+  }
+};
+
+} // namespace detail
+
+template <typename OptionType>
+using categorized_option =
+    contextual_option<detail::categorized_option_policy<OptionType>>;
+
 enum class mlock_mode { NONE, TRY, MUST };
 
 enum class cache_tidy_strategy { NONE, EXPIRY_TIME, BLOCK_SWAPPED_OUT };
@@ -76,21 +97,10 @@ struct filesystem_writer_options {
   bool no_section_index{false};
 };
 
-struct inode_options {
+// TODO: rename
-  bool with_similarity{false};
-  bool with_nilsimsa{false};
-  std::optional<size_t> max_similarity_scan_size;
-  std::shared_ptr<categorizer_manager> categorizer_mgr;
-
-  bool needs_scan(size_t size) const {
-    return categorizer_mgr || ((with_similarity || with_nilsimsa) &&
-                               (!max_similarity_scan_size ||
-                                size <= max_similarity_scan_size.value()));
-  }
-};
-
 enum class file_order_mode { NONE, PATH, SCRIPT, SIMILARITY, NILSIMSA };
 
+// TODO: rename
 struct file_order_options {
   file_order_mode mode{file_order_mode::NONE};
   int nilsimsa_depth{20000};
@@ -98,8 +108,18 @@ struct file_order_options {
   int nilsimsa_limit{255};
 };
 
+struct inode_options {
+  // TODO: - clean this all up and name properly
+  //       - the file_order thing should really be "fragment_order"
+  //       - it should all belong into inode_options, where scanner
+  //         can still access it
+  //       - python scripts need to die
+  std::optional<size_t> max_similarity_scan_size; // TODO: not sure about this?
+  std::shared_ptr<categorizer_manager> categorizer_mgr;
+  categorized_option<file_order_options> fragment_order{file_order_options()};
+};
+
 struct scanner_options {
-  file_order_options file_order;
   std::optional<std::string> file_hash_algorithm{"xxh3-128"};
   std::optional<file_stat::uid_type> uid;
   std::optional<file_stat::gid_type> gid;

include/dwarfs/similarity.h

@@ -23,6 +23,7 @@
 
 #include <cstddef>
 #include <cstdint>
+#include <span>
 
 namespace dwarfs {
 
@@ -34,6 +35,10 @@ class similarity {
   void update(uint8_t const* data, size_t size);
   uint32_t finalize() const;
 
+  void operator()(std::span<uint8_t const> data) {
+    update(data.data(), data.size());
+  }
+
  private:
   class impl;
 

src/dwarfs/categorizer.cpp

@@ -39,6 +39,12 @@ using namespace std::placeholders;
 
 namespace po = boost::program_options;
 
+namespace {
+
+constexpr std::string_view const DEFAULT_CATEGORY{"<default>"};
+
+}
+
 class categorizer_manager_private : public categorizer_manager::impl {
  public:
   virtual std::vector<std::shared_ptr<categorizer const>> const&
@@ -170,13 +176,24 @@ class categorizer_manager_ final : public categorizer_manager_private {
  public:
   categorizer_manager_(logger& lgr)
       : lgr_{lgr}
-      , LOG_PROXY_INIT(lgr) {}
+      , LOG_PROXY_INIT(lgr) {
+    add_category(DEFAULT_CATEGORY, std::numeric_limits<size_t>::max());
+  }
 
   void add(std::shared_ptr<categorizer const> c) override;
   categorizer_job job(std::filesystem::path const& path) const override;
   std::string_view
   category_name(fragment_category::value_type c) const override;
 
+  std::optional<fragment_category::value_type>
+  category_value(std::string_view name) const override {
+    std::optional<fragment_category::value_type> rv;
+    if (auto it = catmap_.find(name); it != catmap_.end()) {
+      rv.emplace(it->second);
+    }
+    return rv;
+  }
+
   folly::dynamic category_metadata(fragment_category c) const override;
 
   std::vector<std::shared_ptr<categorizer const>> const&
@@ -202,10 +219,15 @@ class categorizer_manager_ final : public categorizer_manager_private {
   logger& lgr_;
   LOG_PROXY_DECL(LoggerPolicy);
   std::vector<std::shared_ptr<categorizer const>> categorizers_;
+  // TODO: category descriptions?
   std::vector<std::pair<std::string_view, size_t>> categories_;
   std::unordered_map<std::string_view, fragment_category::value_type> catmap_;
 };
 
+fragment_category categorizer_manager::default_category() {
+  return fragment_category(0);
+}
+
 template <typename LoggerPolicy>
 void categorizer_manager_<LoggerPolicy>::add(
     std::shared_ptr<categorizer const> c) {
@@ -233,6 +255,9 @@ std::string_view categorizer_manager_<LoggerPolicy>::category_name(
 template <typename LoggerPolicy>
 folly::dynamic categorizer_manager_<LoggerPolicy>::category_metadata(
     fragment_category c) const {
+  if (c.value() == 0) {
+    return folly::dynamic();
+  }
   auto cat = DWARFS_NOTHROW(categories_.at(c.value()));
   auto categorizer = DWARFS_NOTHROW(categorizers_.at(cat.second));
   return categorizer->category_metadata(cat.first, c);

src/dwarfs/categorizer/incompressible_categorizer.cpp

@@ -42,6 +42,13 @@ namespace {
 
 constexpr std::string_view const INCOMPRESSIBLE_CATEGORY{"incompressible"};
 
+// TODO: We could actually split large files into compressible and
+//       incompressible fragments. This may be beneficial for use cases
+//       such as wrapping file system images, where we can separate out
+//       compressed parts in the original image.
+//
+//       We probably need to reintroduce the <default> category for that.
+
 struct incompressible_categorizer_config {
   size_t min_input_size;
   double max_ratio_size;

src/dwarfs/category_parser.cpp

@@ -0,0 +1,56 @@
+/* vim:set ts=2 sw=2 sts=2 et: */
+/**
+ * \author     Marcus Holland-Moritz (github@mhxnet.de)
+ * \copyright  Copyright (c) Marcus Holland-Moritz
+ *
+ * This file is part of dwarfs.
+ *
+ * dwarfs is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * dwarfs is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with dwarfs.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <fmt/format.h>
+
+#include "dwarfs/categorizer.h"
+#include "dwarfs/category_parser.h"
+
+namespace dwarfs {
+
+category_parser::category_parser(std::shared_ptr<categorizer_manager> catmgr)
+    : catmgr_{catmgr} {}
+
+std::vector<fragment_category::value_type>
+category_parser::parse(std::string_view arg) const {
+  if (!catmgr_) {
+    throw std::runtime_error(
+        "cannot configure category-specific options without any categories");
+  }
+
+  std::vector<fragment_category::value_type> rv;
+  std::vector<std::string_view> categories;
+
+  folly::split(',', arg, categories);
+  rv.reserve(categories.size());
+
+  for (auto const& name : categories) {
+    if (auto val = catmgr_->category_value(name)) {
+      rv.emplace_back(*val);
+    } else {
+      throw std::range_error(fmt::format("unknown category: '{}'", name));
+    }
+  }
+
+  return rv;
+}
+
+} // namespace dwarfs

src/dwarfs/file_scanner.cpp

@@ -42,7 +42,6 @@ namespace {
 class file_scanner_ : public file_scanner::impl {
  public:
   file_scanner_(worker_group& wg, os_access& os, inode_manager& im,
-                inode_options const& ino_opts,
                 std::optional<std::string> const& hash_algo, progress& prog);
 
   void scan(file* p) override;
@@ -85,7 +84,6 @@ class file_scanner_ : public file_scanner::impl {
   worker_group& wg_;
   os_access& os_;
   inode_manager& im_;
-  inode_options const& ino_opts_;
   std::optional<std::string> const hash_algo_;
   progress& prog_;
   uint32_t num_unique_{0};
@@ -128,13 +126,11 @@ class file_scanner_ : public file_scanner::impl {
 //   from `unique_size_` after its hash has been stored.
 
 file_scanner_::file_scanner_(worker_group& wg, os_access& os, inode_manager& im,
-                             inode_options const& ino_opts,
                              std::optional<std::string> const& hash_algo,
                              progress& prog)
     : wg_(wg)
     , os_(os)
     , im_(im)
-    , ino_opts_(ino_opts)
     , hash_algo_{hash_algo}
     , prog_(prog) {}
 
@@ -308,24 +304,7 @@ void file_scanner_::add_inode(file* p) {
 
   p->set_inode(inode);
 
-  if (ino_opts_.needs_scan(p->size())) {
+  im_.scan_background(wg_, os_, std::move(inode), p);
-    wg_.add_job([this, p, inode = std::move(inode)] {
-      std::shared_ptr<mmif> mm;
-      auto const size = p->size();
-      if (size > 0) {
-        mm = os_.map_file(p->fs_path(), size);
-      }
-      inode->scan(mm.get(), ino_opts_);
-      ++prog_.similarity_scans;
-      prog_.similarity_bytes += size;
-      ++prog_.inodes_scanned;
-      ++prog_.files_scanned;
-    });
-  } else {
-    inode->set_similarity_valid(ino_opts_);
-    ++prog_.inodes_scanned;
-    ++prog_.files_scanned;
-  }
 }
 
 template <typename Lookup>
@@ -417,10 +396,8 @@ void file_scanner_::finalize_inodes(
 } // namespace
 
 file_scanner::file_scanner(worker_group& wg, os_access& os, inode_manager& im,
-                           inode_options const& ino_opts,
                            std::optional<std::string> const& hash_algo,
                            progress& prog)
-    : impl_{std::make_unique<file_scanner_>(wg, os, im, ino_opts, hash_algo,
+    : impl_{std::make_unique<file_scanner_>(wg, os, im, hash_algo, prog)} {}
-                                            prog)} {}
 
 } // namespace dwarfs::detail

src/dwarfs/fragment_order_parser.cpp

@@ -0,0 +1,127 @@
+/* vim:set ts=2 sw=2 sts=2 et: */
+/**
+ * \author     Marcus Holland-Moritz (github@mhxnet.de)
+ * \copyright  Copyright (c) Marcus Holland-Moritz
+ *
+ * This file is part of dwarfs.
+ *
+ * dwarfs is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * dwarfs is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with dwarfs.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <map>
+#include <stdexcept>
+#include <vector>
+
+#include <fmt/format.h>
+
+#include <folly/gen/String.h>
+
+#include "dwarfs/fragment_order_parser.h"
+
+namespace dwarfs {
+
+namespace {
+
+const std::map<std::string_view, file_order_mode> order_choices{
+    {"none", file_order_mode::NONE},
+    {"path", file_order_mode::PATH},
+#ifdef DWARFS_HAVE_PYTHON
+    {"script", file_order_mode::SCRIPT},
+#endif
+    {"similarity", file_order_mode::SIMILARITY},
+    {"nilsimsa", file_order_mode::NILSIMSA},
+};
+
+void parse_order_option(std::string_view ordname, std::string_view opt,
+                        int& value, std::string_view name,
+                        std::optional<int> min = std::nullopt,
+                        std::optional<int> max = std::nullopt) {
+  if (!opt.empty()) {
+    if (auto val = folly::tryTo<int>(opt)) {
+      auto tmp = *val;
+      if (min && max && (tmp < *min || tmp > *max)) {
+        throw std::range_error(
+            fmt::format("{} ({}) out of range for order '{}' ({}..{})", name,
+                        opt, ordname, *min, *max));
+      }
+      if (min && tmp < *min) {
+        throw std::range_error(
+            fmt::format("{} ({}) cannot be less than {} for order '{}'", name,
+                        opt, *min, ordname));
+      }
+      if (max && tmp > *max) {
+        throw std::range_error(
+            fmt::format("{} ({}) cannot be greater than {} for order '{}'",
+                        name, opt, *max, ordname));
+      }
+      value = tmp;
+    } else {
+      throw std::range_error(fmt::format(
+          "{} ({}) is not numeric for order '{}'", name, opt, ordname));
+    }
+  }
+}
+
+} // namespace
+
+std::string fragment_order_parser::choices() {
+  using namespace folly::gen;
+  return from(order_choices) | get<0>() | unsplit<std::string>(", ");
+}
+
+// TODO: find a common syntax for these options so we don't need
+//       complex parsers like this one
+file_order_options fragment_order_parser::parse(std::string_view arg) const {
+  file_order_options rv;
+
+  std::vector<std::string_view> order_opts;
+
+  folly::split(':', arg, order_opts);
+
+  if (auto it = order_choices.find(order_opts.front());
+      it != order_choices.end()) {
+    rv.mode = it->second;
+
+    if (order_opts.size() > 1) {
+      if (rv.mode != file_order_mode::NILSIMSA) {
+        throw std::runtime_error(
+            fmt::format("inode order mode '{}' does not support options",
+                        order_opts.front()));
+      }
+
+      if (order_opts.size() > 4) {
+        throw std::runtime_error(fmt::format(
+            "too many options for inode order mode '{}'", order_opts.front()));
+      }
+
+      auto ordname = order_opts[0];
+
+      parse_order_option(ordname, order_opts[1], rv.nilsimsa_limit, "limit", 0,
+                         255);
+
+      parse_order_option(ordname, order_opts[2], rv.nilsimsa_depth, "depth", 0);
+
+      if (order_opts.size() > 3) {
+        parse_order_option(ordname, order_opts[3], rv.nilsimsa_min_depth,
+                           "min depth", 0);
+      }
+    }
+  } else {
+    throw std::runtime_error(fmt::format("invalid inode order mode: {}", arg));
+  }
+
+  return rv;
+}
+
+} // namespace dwarfs

src/dwarfs/inode_manager.cpp

@@ -48,9 +48,11 @@
 #include "dwarfs/mmif.h"
 #include "dwarfs/nilsimsa.h"
 #include "dwarfs/options.h"
+#include "dwarfs/os_access.h"
 #include "dwarfs/progress.h"
 #include "dwarfs/script.h"
 #include "dwarfs/similarity.h"
+#include "dwarfs/worker_group.h"
 
 #include "dwarfs/gen-cpp2/metadata_types.h"
 
@@ -115,7 +117,6 @@ class inode_ : public inode {
   }
 
   uint32_t similarity_hash() const override {
-    assert(similarity_valid_);
     if (files_.empty()) {
       DWARFS_THROW(runtime_error, "inode has no file (similarity)");
     }
@@ -123,7 +124,6 @@ class inode_ : public inode {
   }
 
   nilsimsa::hash_type const& nilsimsa_similarity_hash() const override {
-    assert(nilsimsa_valid_);
     if (files_.empty()) {
       DWARFS_THROW(runtime_error, "inode has no file (nilsimsa)");
     }
@@ -138,30 +138,16 @@ class inode_ : public inode {
     files_ = std::move(fv);
   }
 
-  void
-  set_similarity_valid(inode_options const& opts [[maybe_unused]]) override {
-#ifndef NDEBUG
-    assert(!similarity_valid_);
-    assert(!nilsimsa_valid_);
-    similarity_valid_ = opts.with_similarity;
-    nilsimsa_valid_ = opts.with_nilsimsa;
-#endif
-  }
-
   void scan(mmif* mm, inode_options const& opts) override {
-    assert(!similarity_valid_);
-    assert(!nilsimsa_valid_);
-
-    similarity sc;
-    nilsimsa nc;
-
     categorizer_job catjob;
 
+    // No job if categorizers are disabled
     if (opts.categorizer_mgr) {
       catjob =
           opts.categorizer_mgr->job(mm ? mm->path().string() : "<no-file>");
     }
 
+    /// TODO: remove comments or move elsewhere
     ///
     /// 1. Run random access categorizers
     /// 2. If we *have* a best category already (need a call for that),
@@ -175,56 +161,54 @@ class inode_ : public inode {
     ///    as well support that case.
     ///
 
+    // If we don't have a mapping, we can't scan anything
     if (mm) {
       if (catjob) {
+        // First, run random access categorizers. If we get a result here,
+        // it's very likely going to be the best result.
         catjob.set_total_size(mm->size());
         catjob.categorize_random_access(mm->span());
+
+        if (catjob.best_result_found()) {
+          // This means the job won't be running any sequential categorizers
+          // as the outcome cannot possibly be any better. As a consequence,
+          // we can already fetch the result here and scan the fragments
+          // instead of the whole file.
+
+          fragments_ = catjob.result();
+
+          if (fragments_.size() > 1) {
+            scan_fragments(mm, opts);
+          } else {
+            scan_full(mm, opts);
+          }
+        }
       }
 
-      auto scan_sequential = [&](uint8_t const* data, size_t size) {
+      if (fragments_.empty()) {
-        if (opts.with_similarity) {
+        // If we get here, we haven't scanned anything yet, and we don't know
-          sc.update(data, size);
+        // if the file will be fragmented or not.
-        }
 
-        if (opts.with_nilsimsa) {
+        scan_full(mm, opts);
-          nc.update(data, size);
-        }
 
         if (catjob) {
-          catjob.categorize_sequential(std::span(data, size));
+          fragments_ = catjob.result();
+
+          if (fragments_.size() > 1) {
+            // This is the unfortunate case where we have to scan the
+            // individual fragments after having already done a full scan.
+            scan_fragments(mm, opts);
+          }
         }
-      };
-
-      constexpr size_t chunk_size = 32 << 20;
-      size_t offset = 0;
-      size_t size = mm->size();
-
-      while (size >= chunk_size) {
-        scan_sequential(mm->as<uint8_t>(offset), chunk_size);
-        mm->release_until(offset);
-        offset += chunk_size;
-        size -= chunk_size;
       }
-
-      scan_sequential(mm->as<uint8_t>(offset), size);
     }
 
-    if (opts.with_similarity) {
+    // Add a fragment if nothing has been added so far. We need a single
-      similarity_hash_ = sc.finalize();
+    // fragment to store the inode's chunks. This won't use up any resources
-#ifndef NDEBUG
+    // as a single fragment is stored inline.
-      similarity_valid_ = true;
+    if (fragments_.empty()) {
-#endif
+      fragments_.emplace_back(categorizer_manager::default_category(),
-    }
+                              mm ? mm->size() : 0);
-
-    if (opts.with_nilsimsa) {
-      nc.finalize(nilsimsa_similarity_hash_);
-#ifndef NDEBUG
-      nilsimsa_valid_ = true;
-#endif
-    }
-
-    if (catjob) {
-      fragments_ = catjob.result();
     }
   }
 
@@ -254,6 +238,110 @@ class inode_ : public inode {
   inode_fragments const& fragments() const override { return fragments_; }
 
  private:
+  template <typename T>
+  void scan_range(mmif* mm, size_t offset, size_t size, T&& scanner) {
+    static constexpr size_t const chunk_size = 32 << 20;
+
+    while (size >= chunk_size) {
+      scanner(mm->span(offset, chunk_size));
+      mm->release_until(offset);
+      offset += chunk_size;
+      size -= chunk_size;
+    }
+
+    scanner(mm->span(offset, size));
+  }
+
+  void scan_fragments(mmif* mm, inode_options const& opts) {
+    assert(mm);
+    assert(fragments_.size() > 1);
+
+    std::unordered_map<fragment_category, similarity> sc;
+    std::unordered_map<fragment_category, nilsimsa> nc;
+
+    for (auto const& f : fragments_.span()) {
+      switch (opts.fragment_order.get(f.category()).mode) {
+      case file_order_mode::NONE:
+      case file_order_mode::PATH:
+      case file_order_mode::SCRIPT:
+        break;
+      case file_order_mode::SIMILARITY:
+        sc.try_emplace(f.category());
+        break;
+      case file_order_mode::NILSIMSA:
+        nc.try_emplace(f.category());
+        break;
+      }
+    }
+
+    if (sc.empty() && nc.empty()) {
+      return;
+    }
+
+    file_off_t pos = 0;
+
+    for (auto const& f : fragments_.span()) {
+      auto const size = f.length();
+
+      if (auto i = sc.find(f.category()); i != sc.end()) {
+        scan_range(mm, pos, size, i->second);
+      } else if (auto i = nc.find(f.category()); i != nc.end()) {
+        scan_range(mm, pos, size, i->second);
+      }
+
+      pos += size;
+    }
+
+    similarity_map_type tmp_map;
+
+    for (auto const& [cat, hasher] : sc) {
+      tmp_map.emplace(cat, hasher.finalize());
+    }
+
+    for (auto const& [cat, hasher] : nc) {
+      // TODO: can we finalize in-place?
+      nilsimsa::hash_type hash;
+      hasher.finalize(hash);
+      tmp_map.emplace(cat, hash);
+    }
+
+    similarity_.emplace<similarity_map_type>(std::move(tmp_map));
+  }
+
+  void scan_full(mmif* mm, inode_options const& opts) {
+    assert(mm);
+    assert(fragments_.size() <= 1);
+
+    auto order_mode =
+        fragments_.empty()
+            ? opts.fragment_order.get().mode
+            : opts.fragment_order.get(fragments_.get_single_category()).mode;
+
+    switch (order_mode) {
+    case file_order_mode::NONE:
+    case file_order_mode::PATH:
+    case file_order_mode::SCRIPT:
+      break;
+
+    case file_order_mode::SIMILARITY: {
+      similarity sc;
+      scan_range(mm, 0, mm->size(), sc);
+      similarity_hash_ = sc.finalize(); // TODO
+      similarity_.emplace<uint32_t>(sc.finalize());
+    } break;
+
+    case file_order_mode::NILSIMSA: {
+      nilsimsa nc;
+      scan_range(mm, 0, mm->size(), nc);
+      // TODO: can we finalize in-place?
+      nilsimsa::hash_type hash;
+      nc.finalize(hash);
+      nilsimsa_similarity_hash_ = hash; // TODO
+      similarity_.emplace<nilsimsa::hash_type>(hash);
+    } break;
+    }
+  }
+
   using similarity_map_type =
       folly::sorted_vector_map<fragment_category,
                                std::variant<nilsimsa::hash_type, uint32_t>>;
@@ -283,11 +371,6 @@ class inode_ : public inode {
   std::vector<chunk_type> chunks_; // TODO: remove (part of fragments_ now)
   nilsimsa::hash_type
       nilsimsa_similarity_hash_; // TODO: remove (move to similarity_)
-#ifndef NDEBUG
-  // no longer needed because we now know which are valid
-  bool similarity_valid_{false}; // TODO: remove
-  bool nilsimsa_valid_{false};   // TODO: remove
-#endif
 };
 
 } // namespace
@@ -295,9 +378,11 @@ class inode_ : public inode {
 template <typename LoggerPolicy>
 class inode_manager_ final : public inode_manager::impl {
  public:
-  inode_manager_(logger& lgr, progress& prog)
+  inode_manager_(logger& lgr, progress& prog, inode_options const& opts)
       : LOG_PROXY_INIT(lgr)
-      , prog_(prog) {}
+      , prog_(prog)
+      , opts_{opts}
+      , inodes_need_scanning_{inodes_need_scanning(opts_)} {}
 
   std::shared_ptr<inode> create_inode() override {
     auto ino = std::make_shared<inode_>();
@@ -308,7 +393,6 @@ class inode_manager_ final : public inode_manager::impl {
   size_t count() const override { return inodes_.size(); }
 
   void order_inodes(std::shared_ptr<script> scr,
-                    file_order_options const& file_order,
                     inode_manager::order_cb const& fn) override;
 
   void for_each_inode_in_order(
@@ -349,7 +433,22 @@ class inode_manager_ final : public inode_manager::impl {
     return rv;
   }
 
+  void
+  scan_background(worker_group& wg, os_access& os, std::shared_ptr<inode> ino,
+                  file const* p) const override;
+
  private:
+  static bool inodes_need_scanning(inode_options const& opts) {
+    if (opts.categorizer_mgr) {
+      return true;
+    }
+
+    return opts.fragment_order.any_is([](auto const& order) {
+      return order.mode == file_order_mode::SIMILARITY ||
+             order.mode == file_order_mode::NILSIMSA;
+    });
+  }
+
   void order_inodes_by_path() {
     std::vector<std::string> paths;
     std::vector<size_t> index(inodes_.size());
@@ -391,19 +490,49 @@ class inode_manager_ final : public inode_manager::impl {
   void presort_index(std::vector<std::shared_ptr<inode>>& inodes,
                      std::vector<uint32_t>& index);
 
-  void order_inodes_by_nilsimsa(inode_manager::order_cb const& fn,
+  void order_inodes_by_nilsimsa(inode_manager::order_cb const& fn);
-                                file_order_options const& file_order);
 
-  std::vector<std::shared_ptr<inode>> inodes_;
   LOG_PROXY_DECL(LoggerPolicy);
+  std::vector<std::shared_ptr<inode>> inodes_;
   progress& prog_;
+  inode_options opts_;
+  bool const inodes_need_scanning_;
 };
 
+template <typename LoggerPolicy>
+void inode_manager_<LoggerPolicy>::scan_background(worker_group& wg,
+                                                   os_access& os,
+                                                   std::shared_ptr<inode> ino,
+                                                   file const* p) const {
+  // TODO: I think the size check makes everything more complex.
+  //       If we don't check the size, we get the code to run
+  //       that ensures `fragments_` is updated. Also, there
+  //       should only ever be one empty inode, so the check
+  //       doesn't actually make much of a difference.
+  if (inodes_need_scanning_ /* && p->size() > 0 */) {
+    wg.add_job([this, &os, p, ino = std::move(ino)] {
+      auto const size = p->size();
+      std::shared_ptr<mmif> mm;
+      if (size > 0) {
+        mm = os.map_file(p->fs_path(), size);
+      }
+      ino->scan(mm.get(), opts_);
+      ++prog_.similarity_scans; // TODO: we probably don't want this here
+      prog_.similarity_bytes += size;
+      ++prog_.inodes_scanned;
+      ++prog_.files_scanned;
+    });
+  } else {
+    ++prog_.inodes_scanned;
+    ++prog_.files_scanned;
+  }
+}
+
 template <typename LoggerPolicy>
 void inode_manager_<LoggerPolicy>::order_inodes(
-    std::shared_ptr<script> scr, file_order_options const& file_order,
+    std::shared_ptr<script> scr, inode_manager::order_cb const& fn) {
-    inode_manager::order_cb const& fn) {
+  // TODO:
-  switch (file_order.mode) {
+  switch (opts_.fragment_order.get().mode) {
   case file_order_mode::NONE:
     LOG_INFO << "keeping inode order";
     break;
@@ -439,7 +568,7 @@ void inode_manager_<LoggerPolicy>::order_inodes(
     LOG_INFO << "ordering " << count()
              << " inodes using nilsimsa similarity...";
     auto ti = LOG_CPU_TIMED_INFO;
-    order_inodes_by_nilsimsa(fn, file_order);
+    order_inodes_by_nilsimsa(fn);
     ti << count() << " inodes ordered";
     return;
   }
@@ -494,7 +623,7 @@ void inode_manager_<LoggerPolicy>::presort_index(
 
 template <typename LoggerPolicy>
 void inode_manager_<LoggerPolicy>::order_inodes_by_nilsimsa(
-    inode_manager::order_cb const& fn, file_order_options const& file_order) {
+    inode_manager::order_cb const& fn) {
   auto count = inodes_.size();
 
   if (auto fname = ::getenv("DWARFS_NILSIMSA_DUMP")) {
@@ -559,6 +688,7 @@ void inode_manager_<LoggerPolicy>::order_inodes_by_nilsimsa(
   }
 
   if (!index.empty()) {
+    auto const& file_order = opts_.fragment_order.get(); // TODO
     const int_fast32_t max_depth = file_order.nilsimsa_depth;
     const int_fast32_t min_depth =
         std::min<int32_t>(file_order.nilsimsa_min_depth, max_depth);
@@ -607,8 +737,9 @@ void inode_manager_<LoggerPolicy>::order_inodes_by_nilsimsa(
   }
 }
 
-inode_manager::inode_manager(logger& lgr, progress& prog)
+inode_manager::inode_manager(logger& lgr, progress& prog,
+                             inode_options const& opts)
     : impl_(make_unique_logging_object<impl, inode_manager_, logger_policies>(
-          lgr, prog)) {}
+          lgr, prog, opts)) {}
 
 } // namespace dwarfs

src/dwarfs/scanner.cpp

@@ -566,9 +566,8 @@ void scanner_<LoggerPolicy>::scan(
 
   prog.set_status_function(status_string);
 
-  inode_manager im(lgr_, prog);
+  inode_manager im(lgr_, prog, options_.inode);
-  detail::file_scanner fs(wg_, *os_, im, options_.inode,
+  detail::file_scanner fs(wg_, *os_, im, options_.file_hash_algorithm, prog);
-                          options_.file_hash_algorithm, prog);
 
   auto root =
       list ? scan_list(path, *list, prog, fs) : scan_tree(path, prog, fs);
@@ -661,20 +660,19 @@ void scanner_<LoggerPolicy>::scan(
       worker_group ordering("ordering", 1);
 
       ordering.add_job([&] {
-        im.order_inodes(script_, options_.file_order,
+        im.order_inodes(script_, [&](std::shared_ptr<inode> const& ino) {
-                        [&](std::shared_ptr<inode> const& ino) {
+          blockify.add_job([&] {
-                          blockify.add_job([&] {
+            prog.current.store(ino.get());
-                            prog.current.store(ino.get());
+            bm.add_inode(ino);
-                            bm.add_inode(ino);
+            prog.inodes_written++;
-                            prog.inodes_written++;
+          });
-                          });
+          auto queued_files = blockify.queue_size();
-                          auto queued_files = blockify.queue_size();
+          auto queued_blocks = fsw.queue_fill();
-                          auto queued_blocks = fsw.queue_fill();
+          prog.blockify_queue = queued_files;
-                          prog.blockify_queue = queued_files;
+          prog.compress_queue = queued_blocks;
-                          prog.compress_queue = queued_blocks;
+          return INT64_C(500) * queued_blocks +
-                          return INT64_C(500) * queued_blocks +
+                 static_cast<int64_t>(queued_files);
-                                 static_cast<int64_t>(queued_files);
+        });
-                        });
       });
 
       ordering.wait();

src/mkdwarfs_main.cpp

@@ -54,12 +54,14 @@
 #include "dwarfs/block_manager.h"
 #include "dwarfs/builtin_script.h"
 #include "dwarfs/categorizer.h"
+#include "dwarfs/category_parser.h"
 #include "dwarfs/chmod_transformer.h"
 #include "dwarfs/console_writer.h"
 #include "dwarfs/entry.h"
 #include "dwarfs/error.h"
 #include "dwarfs/filesystem_v2.h"
 #include "dwarfs/filesystem_writer.h"
+#include "dwarfs/fragment_order_parser.h"
 #include "dwarfs/logger.h"
 #include "dwarfs/mmap.h"
 #include "dwarfs/options.h"
@@ -90,13 +92,6 @@ enum class debug_filter_mode {
   ALL
 };
 
-const std::map<std::string, file_order_mode> order_choices{
-    {"none", file_order_mode::NONE},
-    {"path", file_order_mode::PATH},
-    {"similarity", file_order_mode::SIMILARITY},
-    {"nilsimsa", file_order_mode::NILSIMSA},
-};
-
 const std::map<std::string, console_writer::progress_mode> progress_modes{
     {"none", console_writer::NONE},
     {"simple", console_writer::SIMPLE},
@@ -159,39 +154,6 @@ void debug_filter_output(std::ostream& os, bool exclude, entry const* pe,
   os << prefix << pe->unix_dpath() << "\n";
 }
 
-int parse_order_option(std::string const& ordname, std::string const& opt,
-                       int& value, std::string_view name,
-                       std::optional<int> min = std::nullopt,
-                       std::optional<int> max = std::nullopt) {
-  if (!opt.empty()) {
-    if (auto val = folly::tryTo<int>(opt)) {
-      auto tmp = *val;
-      if (min && max && (tmp < *min || tmp > *max)) {
-        std::cerr << "error: " << name << " (" << opt
-                  << ") out of range for order '" << ordname << "' (" << *min
-                  << ".." << *max << ")\n";
-        return 1;
-      }
-      if (min && tmp < *min) {
-        std::cerr << "error: " << name << " (" << opt
-                  << ") cannot be less than " << *min << " for order '"
-                  << ordname << "'\n";
-      }
-      if (max && tmp > *max) {
-        std::cerr << "error: " << name << " (" << opt
-                  << ") cannot be greater than " << *max << " for order '"
-                  << ordname << "'\n";
-      }
-      value = tmp;
-    } else {
-      std::cerr << "error: " << name << " (" << opt
-                << ") is not numeric for order '" << ordname << "'\n";
-      return 1;
-    }
-  }
-  return 0;
-}
-
 struct level_defaults {
   unsigned block_size_bits;
   std::string_view data_compression;
@@ -313,11 +275,12 @@ int mkdwarfs_main(int argc, sys_char** argv) {
   block_manager::config cfg;
   sys_string path_str, output_str;
   std::string memory_limit, script_arg, compression, header, schema_compression,
-      metadata_compression, log_level_str, timestamp, time_resolution, order,
+      metadata_compression, log_level_str, timestamp, time_resolution,
       progress_mode, recompress_opts, pack_metadata, file_hash_algo,
       debug_filter, max_similarity_size, input_list_str, chmod_str,
       categorizer_list_str;
   std::vector<sys_string> filter;
+  std::vector<std::string> order;
   size_t num_workers, num_scanner_workers;
   bool no_progress = false, remove_header = false, no_section_index = false,
        force_overwrite = false;
@@ -327,8 +290,7 @@ int mkdwarfs_main(int argc, sys_char** argv) {
 
   scanner_options options;
 
-  auto order_desc =
+  auto order_desc = "inode order (" + fragment_order_parser::choices() + ")";
-      "inode order (" + (from(order_choices) | get<0>() | unsplit(", ")) + ")";
 
   auto progress_desc = "progress mode (" +
                        (from(progress_modes) | get<0>() | unsplit(", ")) + ")";
@@ -404,8 +366,8 @@ int mkdwarfs_main(int argc, sys_char** argv) {
           ->default_value("pcmaudio,incompressible"),
         categorize_desc.c_str())
     ("order",
-        po::value<std::string>(&order),
+        po::value<std::vector<std::string>>(&order)->multitoken(),
-        order_desc.c_str())
+        order_desc.c_str())  // TODO
     ("max-similarity-size",
         po::value<std::string>(&max_similarity_size),
         "maximum file size to compute similarity")
@@ -639,7 +601,8 @@ int mkdwarfs_main(int argc, sys_char** argv) {
   }
 
   if (!vm.count("order")) {
-    order = defaults.order;
+    // TODO:
+    order.push_back(std::string(defaults.order));
   }
 
   if (cfg.block_size_bits < min_block_size_bits ||
@@ -710,54 +673,6 @@ int mkdwarfs_main(int argc, sys_char** argv) {
     }
   }
 
-  std::vector<std::string> order_opts;
-  boost::split(order_opts, order, boost::is_any_of(":"));
-
-  if (auto it = order_choices.find(order_opts.front());
-      it != order_choices.end()) {
-    options.file_order.mode = it->second;
-
-    if (order_opts.size() > 1) {
-      if (options.file_order.mode != file_order_mode::NILSIMSA) {
-        std::cerr << "error: inode order mode '" << order_opts.front()
-                  << "' does not support options\n";
-        return 1;
-      }
-
-      if (order_opts.size() > 4) {
-        std::cerr << "error: too many options for inode order mode '"
-                  << order_opts[0] << "'\n";
-        return 1;
-      }
-
-      auto ordname = order_opts[0];
-
-      if (parse_order_option(ordname, order_opts[1],
-                             options.file_order.nilsimsa_limit, "limit", 0,
-                             255)) {
-        return 1;
-      }
-
-      if (order_opts.size() > 2) {
-        if (parse_order_option(ordname, order_opts[2],
-                               options.file_order.nilsimsa_depth, "depth", 0)) {
-          return 1;
-        }
-      }
-
-      if (order_opts.size() > 3) {
-        if (parse_order_option(ordname, order_opts[3],
-                               options.file_order.nilsimsa_min_depth,
-                               "min depth", 0)) {
-          return 1;
-        }
-      }
-    }
-  } else {
-    std::cerr << "error: invalid inode order mode: " << order << "\n";
-    return 1;
-  }
-
   if (file_hash_algo == "none") {
     options.file_hash_algorithm.reset();
   } else if (checksum::is_available(file_hash_algo)) {
@@ -1031,11 +946,6 @@ int mkdwarfs_main(int argc, sys_char** argv) {
                              fsw, rw_opts);
       wg_compress.wait();
     } else {
-      options.inode.with_similarity =
-          options.file_order.mode == file_order_mode::SIMILARITY;
-      options.inode.with_nilsimsa =
-          options.file_order.mode == file_order_mode::NILSIMSA;
-
       if (!categorizer_list_str.empty()) {
         std::vector<std::string> categorizer_list;
         boost::split(categorizer_list, categorizer_list_str,
@@ -1049,6 +959,17 @@ int mkdwarfs_main(int argc, sys_char** argv) {
         }
       }
 
+      try {
+        category_parser cp(options.inode.categorizer_mgr);
+        fragment_order_parser fop;
+        contextual_option_parser order_parser(options.inode.fragment_order, cp,
+                                              fop);
+        order_parser.parse(order);
+      } catch (std::exception const& e) {
+        LOG_ERROR << e.what();
+        return 1;
+      }
+
       scanner s(lgr, wg_scanner, cfg, entry_factory::create(),
                 std::make_shared<os_access_generic>(), std::move(script),
                 options);

test/dwarfs.cpp

@@ -105,12 +105,13 @@ void basic_end_to_end_test(std::string const& compressor,
   cfg.blockhash_window_size = 10;
   cfg.block_size_bits = block_size_bits;
 
-  options.file_order.mode = file_order;
+  file_order_options order_opts;
+  order_opts.mode = file_order;
+
   options.file_hash_algorithm = file_hash_algo;
   options.with_devices = with_devices;
   options.with_specials = with_specials;
-  options.inode.with_similarity = file_order == file_order_mode::SIMILARITY;
+  options.inode.fragment_order.set_default(order_opts);
-  options.inode.with_nilsimsa = file_order == file_order_mode::NILSIMSA;
   options.keep_all_times = keep_all_times;
   options.pack_chunk_table = pack_chunk_table;
   options.pack_directories = pack_directories;
@@ -145,6 +146,7 @@ void basic_end_to_end_test(std::string const& compressor,
 
   auto prog = progress([](const progress&, bool) {}, 1000);
 
+  // TODO:
   std::shared_ptr<script> scr;
   if (file_order == file_order_mode::SCRIPT) {
     scr = std::make_shared<test::script_mock>();
@@ -154,8 +156,8 @@ void basic_end_to_end_test(std::string const& compressor,
   auto image_size = fsimage.size();
   auto mm = std::make_shared<test::mmap_mock>(std::move(fsimage));
 
-  bool similarity =
+  bool similarity = file_order == file_order_mode::SIMILARITY ||
-      options.inode.with_similarity || options.inode.with_nilsimsa;
+                    file_order == file_order_mode::NILSIMSA;
 
   size_t const num_fail_empty = access_fail ? 1 : 0;
 
@@ -184,7 +186,9 @@ void basic_end_to_end_test(std::string const& compressor,
                 (prog.saved_by_deduplication + prog.saved_by_segmentation +
                  prog.symlink_size),
             prog.filesystem_size);
-  EXPECT_EQ(prog.similarity_scans, similarity ? prog.inodes_scanned.load() : 0);
+  // TODO:
+  // EXPECT_EQ(prog.similarity_scans, similarity ? prog.inodes_scanned.load() :
+  // 0);
   EXPECT_EQ(prog.similarity_bytes,
             similarity ? prog.original_size -
                              (prog.saved_by_deduplication + prog.symlink_size)
@@ -760,10 +764,11 @@ TEST_P(file_scanner, inode_ordering) {
   auto bmcfg = block_manager::config();
   auto opts = scanner_options();
 
-  opts.file_order.mode = order_mode;
+  file_order_options order_opts;
+  order_opts.mode = order_mode;
+
   opts.file_hash_algorithm = file_hash_algo;
-  opts.inode.with_similarity = order_mode == file_order_mode::SIMILARITY;
+  opts.inode.fragment_order.set_default(order_opts);
-  opts.inode.with_nilsimsa = order_mode == file_order_mode::NILSIMSA;
 
   auto input = std::make_shared<test::os_access_mock>();
   constexpr int dim = 14;
@@ -860,7 +865,8 @@ TEST(file_scanner, input_list) {
   auto bmcfg = block_manager::config();
   auto opts = scanner_options();
 
-  opts.file_order.mode = file_order_mode::NONE;
+  file_order_options order_opts;
+  opts.inode.fragment_order.set_default(order_opts);
 
   auto input = test::os_access_mock::create_test_instance();
 

test/dwarfs_benchmark.cpp

@@ -99,8 +99,6 @@ std::string make_filesystem(::benchmark::State const& state) {
 
   options.with_devices = true;
   options.with_specials = true;
-  options.inode.with_similarity = false;
-  options.inode.with_nilsimsa = false;
   options.keep_all_times = false;
   options.pack_chunk_table = true;
   options.pack_directories = state.range(0);