diff --git a/test/dwarfs_benchmark.cpp b/test/dwarfs_benchmark.cpp index af861d92..e459b020 100644 --- a/test/dwarfs_benchmark.cpp +++ b/test/dwarfs_benchmark.cpp @@ -19,6 +19,7 @@ * along with dwarfs. If not, see . */ +#include #include #include @@ -30,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -95,7 +97,10 @@ void PackParamsDirs(::benchmark::internal::Benchmark* b) { } } -std::string make_filesystem(::benchmark::State const& state) { +std::string +make_filesystem(::benchmark::State const* state, + std::shared_ptr os = nullptr, + writer::segmenter_factory::config const* segcfg = nullptr) { writer::segmenter_factory::config cfg; writer::scanner_options options; @@ -105,27 +110,33 @@ std::string make_filesystem(::benchmark::State const& state) { cfg.bloom_filter_size.set_default(4); cfg.block_size_bits = 12; + options.inode.fragment_order.set_default( + {.mode = writer::fragment_order_mode::PATH}); + options.with_devices = true; options.with_specials = true; options.keep_all_times = false; options.pack_chunk_table = true; - options.pack_directories = state.range(0); + options.pack_directories = state ? state->range(0) : true; options.pack_shared_files_table = true; - options.pack_names = state.range(2); - options.pack_names_index = state.range(3); - options.pack_symlinks = state.range(2); - options.pack_symlinks_index = state.range(3); + options.pack_names = state ? state->range(2) : true; + options.pack_names_index = state ? state->range(3) : true; + options.pack_symlinks = state ? state->range(2) : true; + options.pack_symlinks_index = state ? state->range(3) : true; options.force_pack_string_tables = true; - options.plain_names_table = state.range(1); - options.plain_symlinks_table = state.range(1); + options.plain_names_table = state ? state->range(1) : false; + options.plain_symlinks_table = state ? state->range(1) : false; test::test_logger lgr; - auto os = test::os_access_mock::create_test_instance(); + + if (!os) { + os = test::os_access_mock::create_test_instance(); + } thread_pool pool(lgr, *os, "writer", 4); writer::writer_progress prog; - writer::segmenter_factory sf(lgr, prog, cfg); + writer::segmenter_factory sf(lgr, prog, segcfg ? *segcfg : cfg); writer::entry_factory ef; writer::scanner s(lgr, pool, sf, ef, *os, options); @@ -184,7 +195,7 @@ void frozen_string_table_lookup(::benchmark::State& state) { } void dwarfs_initialize(::benchmark::State& state) { - auto image = make_filesystem(state); + auto image = make_filesystem(&state); test::test_logger lgr; test::os_access_mock os; auto mm = std::make_shared(image); @@ -203,7 +214,7 @@ class filesystem : public ::benchmark::Fixture { static constexpr size_t NUM_ENTRIES = 8; void SetUp(::benchmark::State const& state) { - image = make_filesystem(state); + image = make_filesystem(&state); mm = std::make_shared(image); reader::filesystem_options opts; opts.block_cache.max_bytes = 1 << 20; @@ -305,6 +316,123 @@ class filesystem : public ::benchmark::Fixture { std::shared_ptr mm; }; +class filesystem_walk : public ::benchmark::Fixture { + public: + void SetUp(::benchmark::State const&) { + mm = std::make_shared(get_image()); + reader::filesystem_options opts; + opts.block_cache.max_bytes = 1 << 20; + opts.metadata.enable_nlink = true; + fs = std::make_unique(lgr, os, mm, opts); + // fs->dump(std::cout, {.features = reader::fsinfo_features::for_level(2)}); + } + + void TearDown(::benchmark::State const&) { + mm.reset(); + fs.reset(); + } + + std::unique_ptr fs; + std::vector entries; + + private: + constexpr static int kDimension{32}; + constexpr static size_t kPatternLength{16}; + + static std::string make_data(std::mt19937_64& rng, size_t size) { + std::string data; + std::uniform_int_distribution<> byte_dist{0, 31}; + data.reserve(size * kPatternLength * 2); + for (size_t i = 0; i < size; ++i) { + char p1 = byte_dist(rng); + char p2 = 128 + byte_dist(rng); + for (size_t j = 0; j < kPatternLength; ++j) { + data.push_back(p1); + data.push_back(p2); + } + } + return data; + } + + static void add_random_file_tree(test::os_access_mock& os) { + std::mt19937_64 rng{42}; + std::uniform_int_distribution<> size_dist{1, 16}; + std::uniform_int_distribution<> path_comp_size_dist{1, 10}; + + auto random_path_component = [&] { + auto size = path_comp_size_dist(rng); + return test::create_random_string(size, 'A', 'Z', rng); + }; + + for (int u = 0; u < kDimension; ++u) { + std::filesystem::path d1{random_path_component() + std::to_string(u)}; + os.add_dir(d1); + + for (int v = 0; v < kDimension; ++v) { + std::filesystem::path d2{d1 / + (random_path_component() + std::to_string(v))}; + os.add_dir(d2); + + for (int w = 0; w < kDimension; ++w) { + std::filesystem::path d3{ + d2 / (random_path_component() + std::to_string(w))}; + os.add_dir(d3); + + for (int z = 0; z < kDimension; ++z) { + std::filesystem::path f{ + d3 / (random_path_component() + std::to_string(z))}; + os.add_file(f, make_data(rng, size_dist(rng))); + } + } + } + } + } + + static std::string build_image() { + auto os = std::make_shared(); + os->add("", {1, 040755, 1, 0, 0, 10, 42, 0, 0, 0}); + add_random_file_tree(*os); + writer::segmenter_factory::config cfg; + cfg.blockhash_window_size.set_default(4); + cfg.window_increment_shift.set_default(1); + cfg.max_active_blocks.set_default(4); + cfg.bloom_filter_size.set_default(4); + cfg.block_size_bits = 16; + return make_filesystem(nullptr, os, &cfg); + } + + static std::string get_image() { + static std::string const image = [] { + std::string image; + if (auto file = std::getenv("DWARFS_BENCHMARK_SAVE_IMAGE")) { + std::cerr << "*** Saving image to " << file << std::endl; + image = build_image(); + std::ofstream ofs(file, std::ios::binary); + ofs.write(image.data(), image.size()); + } else if (auto file = std::getenv("DWARFS_BENCHMARK_LOAD_IMAGE")) { + std::cerr << "*** Loading image from " << file << std::endl; + std::ifstream ifs(file, std::ios::binary); + if (ifs) { + ifs.seekg(0, std::ios::end); + image.resize(ifs.tellg()); + ifs.seekg(0, std::ios::beg); + ifs.read(image.data(), image.size()); + } else { + throw std::runtime_error("Failed to open image file"); + } + } else { + image = build_image(); + } + return image; + }(); + return image; + } + + test::test_logger lgr; + test::os_access_mock os; + std::shared_ptr mm; +}; + BENCHMARK_DEFINE_F(filesystem, find_path)(::benchmark::State& state) { std::array paths{{ "/test.pl", @@ -510,6 +638,19 @@ BENCHMARK_DEFINE_F(filesystem, readv_future_large)(::benchmark::State& state) { readv_future_bench(state, "/ipsum.txt"); } +BENCHMARK_DEFINE_F(filesystem_walk, walk)(::benchmark::State& state) { + for (auto _ : state) { + fs->walk([](reader::dir_entry_view) {}); + } +} + +BENCHMARK_DEFINE_F(filesystem_walk, walk_data_order) +(::benchmark::State& state) { + for (auto _ : state) { + fs->walk_data_order([](reader::dir_entry_view) {}); + } +} + } // namespace BENCHMARK(frozen_legacy_string_table_lookup); @@ -553,4 +694,8 @@ BENCHMARK_REGISTER_F(filesystem, readv_large)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, readv_future_small)->Apply(PackParamsNone); BENCHMARK_REGISTER_F(filesystem, readv_future_large)->Apply(PackParamsNone); +BENCHMARK_REGISTER_F(filesystem_walk, walk)->Unit(benchmark::kMillisecond); +BENCHMARK_REGISTER_F(filesystem_walk, walk_data_order) + ->Unit(benchmark::kMillisecond); + BENCHMARK_MAIN();