TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-12 22:10:54 -04:00
Code Issues Packages Projects Releases Wiki Activity

Remove script ordering and more cleaning up

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2023-08-13 07:18:59 +02:00
parent 51b1c1148a
commit e1a26d77c8
7 changed files with 29 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/dwarfs/inode_manager.h
View File

@ -55,9 +55,8 @@ class inode_manager {
size_t count() const { return impl_->count(); }
void order_inodes(worker_group& wg, std::shared_ptr<script> scr,
order_cb const& fn) {
impl_->order_inodes(wg, std::move(scr), fn);
void order_inodes(worker_group& wg, order_cb const& fn) {
impl_->order_inodes(wg, fn);
}
void for_each_inode_in_order(inode_cb const& fn) const {
@ -88,8 +87,7 @@ class inode_manager {
virtual std::shared_ptr<inode> create_inode() = 0;
virtual size_t count() const = 0;
virtual void order_inodes(worker_group& wg, std::shared_ptr<script> scr,
order_cb const& fn) = 0;
virtual void order_inodes(worker_group& wg, order_cb const& fn) = 0;
virtual void for_each_inode_in_order(
std::function<void(std::shared_ptr<inode> const&)> const& fn) const = 0;
virtual std::vector<std::pair<fragment_category::value_type, size_t>>

2
include/dwarfs/options.h
View File

@ -98,7 +98,7 @@ struct filesystem_writer_options {
};
// TODO: rename
enum class file_order_mode { NONE, PATH, SCRIPT, SIMILARITY, NILSIMSA };
enum class file_order_mode { NONE, PATH, SIMILARITY, NILSIMSA };
// TODO: rename
struct file_order_options {

3
src/dwarfs/fragment_order_parser.cpp
View File

@ -133,9 +133,6 @@ fragment_order_parser::to_string(file_order_options const& opts) const {
case file_order_mode::PATH:
return "path";
case file_order_mode::SCRIPT:
return "script";
case file_order_mode::SIMILARITY:
return "similarity";

20
src/dwarfs/inode_manager.cpp
View File

@ -340,7 +340,6 @@ class inode_ : public inode {
switch (opts.fragment_order.get(f.category()).mode) {
case file_order_mode::NONE:
case file_order_mode::PATH:
case file_order_mode::SCRIPT:
break;
case file_order_mode::SIMILARITY:
sc.try_emplace(f.category());
@ -397,7 +396,6 @@ class inode_ : public inode {
switch (order_mode) {
case file_order_mode::NONE:
case file_order_mode::PATH:
case file_order_mode::SCRIPT:
break;
case file_order_mode::SIMILARITY: {
@ -469,8 +467,8 @@ class inode_manager_ final : public inode_manager::impl {
size_t count() const override { return inodes_.size(); }
void order_inodes(worker_group& wg, std::shared_ptr<script> scr,
inode_manager::order_cb const& fn) override;
void
order_inodes(worker_group& wg, inode_manager::order_cb const& fn) override;
void for_each_inode_in_order(
std::function<void(std::shared_ptr<inode> const&)> const& fn)
@ -608,8 +606,7 @@ void inode_manager_<LoggerPolicy>::scan_background(worker_group& wg,
template <typename LoggerPolicy>
void inode_manager_<LoggerPolicy>::order_inodes(
worker_group& wg, std::shared_ptr<script> scr,
inode_manager::order_cb const& fn) {
worker_group& wg, inode_manager::order_cb const& fn) {
// TODO: only use an index, never actually reorder inodes
// TODO:
@ -626,17 +623,6 @@ void inode_manager_<LoggerPolicy>::order_inodes(
break;
}
case file_order_mode::SCRIPT: {
if (!scr->has_order()) {
DWARFS_THROW(runtime_error, "script cannot order inodes");
}
LOG_INFO << "ordering " << count() << " inodes using script...";
auto ti = LOG_CPU_TIMED_INFO;
scr->order(inodes_);
ti << count() << " inodes ordered";
break;
}
case file_order_mode::SIMILARITY: {
LOG_INFO << "ordering " << count() << " inodes by similarity...";
auto ti = LOG_CPU_TIMED_INFO;

3
src/dwarfs/options.cpp
View File

@ -39,9 +39,6 @@ std::ostream& operator<<(std::ostream& os, file_order_mode mode) {
case file_order_mode::PATH:
modestr = "path";
break;
case file_order_mode::SCRIPT:
modestr = "script";
break;
case file_order_mode::SIMILARITY:
modestr = "similarity";
break;

29
src/dwarfs/scanner.cpp
View File

@ -670,21 +670,20 @@ void scanner_<LoggerPolicy>::scan(
worker_group wg_order("ordering", num_threads);
// ordering.add_job([&] {
im.order_inodes(wg_order, script_,
[&](std::shared_ptr<inode> const& ino) {
blockify.add_job([&, this] {
prog.current.store(ino.get());
inode_chunkable ic(*ino, *os_);
seg.add_chunkable(ic);
prog.inodes_written++;
});
auto queued_files = blockify.queue_size();
auto queued_blocks = fsw.queue_fill();
prog.blockify_queue = queued_files;
prog.compress_queue = queued_blocks;
return INT64_C(500) * queued_blocks +
static_cast<int64_t>(queued_files);
});
im.order_inodes(wg_order, [&](std::shared_ptr<inode> const& ino) {
blockify.add_job([&, this] {
prog.current.store(ino.get());
inode_chunkable ic(*ino, *os_);
seg.add_chunkable(ic);
prog.inodes_written++;
});
auto queued_files = blockify.queue_size();
auto queued_blocks = fsw.queue_fill();
prog.blockify_queue = queued_files;
prog.compress_queue = queued_blocks;
return INT64_C(500) * queued_blocks +
static_cast<int64_t>(queued_files);
});
// });
// wg_order.wait();

19
test/dwarfs.cpp
View File

@ -146,11 +146,7 @@ void basic_end_to_end_test(std::string const& compressor,
auto prog = progress([](const progress&, bool) {}, 1000);
// TODO:
std::shared_ptr<script> scr;
if (file_order == file_order_mode::SCRIPT) {
scr = std::make_shared<test::script_mock>();
}
auto scr = std::make_shared<test::script_mock>();
auto fsimage = build_dwarfs(lgr, input, compressor, cfg, options, &prog, scr);
auto image_size = fsimage.size();
@ -595,12 +591,13 @@ TEST_P(packing_test, regression_empty_fs) {
INSTANTIATE_TEST_SUITE_P(
dwarfs, compression_test,
::testing::Combine(
::testing::ValuesIn(compressions), ::testing::Values(12, 15, 20, 28),
::testing::Values(file_order_mode::NONE, file_order_mode::PATH,
file_order_mode::SCRIPT, file_order_mode::NILSIMSA,
file_order_mode::SIMILARITY),
::testing::Values(std::nullopt, "xxh3-128")));
::testing::Combine(::testing::ValuesIn(compressions),
::testing::Values(12, 15, 20, 28),
::testing::Values(file_order_mode::NONE,
file_order_mode::PATH,
file_order_mode::NILSIMSA,
file_order_mode::SIMILARITY),
::testing::Values(std::nullopt, "xxh3-128")));
INSTANTIATE_TEST_SUITE_P(
dwarfs, scanner_test,