TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-08 20:12:56 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(filesystem_v2): support maxiov parameter for readv()

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2024-08-19 10:44:44 +02:00
parent 9724cf4203
commit d0c5609172
4 changed files with 133 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
include/dwarfs/reader/filesystem_v2.h
View File

@ -245,6 +245,11 @@ class filesystem_v2 {
return impl_->readv(inode, buf, size, offset, ec);
}
size_t readv(uint32_t inode, iovec_read_buf& buf, size_t size,
file_off_t offset, size_t maxiov, std::error_code& ec) const {
return impl_->readv(inode, buf, size, offset, maxiov, ec);
}
size_t readv(uint32_t inode, iovec_read_buf& buf, size_t size,
std::error_code& ec) const {
return impl_->readv(inode, buf, size, 0, ec);
@ -255,6 +260,11 @@ class filesystem_v2 {
return impl_->readv(inode, buf, size, offset);
}
size_t readv(uint32_t inode, iovec_read_buf& buf, size_t size,
file_off_t offset, size_t maxiov) const {
return impl_->readv(inode, buf, size, offset, maxiov);
}
std::vector<std::future<block_range>> readv(uint32_t inode) const {
return impl_->readv(inode);
}
@ -269,6 +279,11 @@ class filesystem_v2 {
return impl_->readv(inode, size, offset);
}
std::vector<std::future<block_range>>
readv(uint32_t inode, size_t size, file_off_t offset, size_t maxiov) const {
return impl_->readv(inode, size, offset, maxiov);
}
std::vector<std::future<block_range>>
readv(uint32_t inode, size_t size, std::error_code& ec) const {
return impl_->readv(inode, size, 0, ec);
@ -280,6 +295,12 @@ class filesystem_v2 {
return impl_->readv(inode, size, offset, ec);
}
std::vector<std::future<block_range>>
readv(uint32_t inode, size_t size, file_off_t offset, size_t maxiov,
std::error_code& ec) const {
return impl_->readv(inode, size, offset, maxiov, ec);
}
std::optional<std::span<uint8_t const>> header() const {
return impl_->header();
}
@ -378,6 +399,11 @@ class filesystem_v2 {
file_off_t offset, std::error_code& ec) const = 0;
virtual size_t readv(uint32_t inode, iovec_read_buf& buf, size_t size,
file_off_t offset) const = 0;
virtual size_t
readv(uint32_t inode, iovec_read_buf& buf, size_t size, file_off_t offset,
size_t maxiov, std::error_code& ec) const = 0;
virtual size_t readv(uint32_t inode, iovec_read_buf& buf, size_t size,
file_off_t offset, size_t maxiov) const = 0;
virtual std::vector<std::future<block_range>>
readv(uint32_t inode) const = 0;
virtual std::vector<std::future<block_range>>
@ -387,6 +413,12 @@ class filesystem_v2 {
virtual std::vector<std::future<block_range>>
readv(uint32_t inode, size_t size, file_off_t offset,
std::error_code& ec) const = 0;
virtual std::vector<std::future<block_range>>
readv(uint32_t inode, size_t size, file_off_t offset,
size_t maxiov) const = 0;
virtual std::vector<std::future<block_range>>
readv(uint32_t inode, size_t size, file_off_t offset, size_t maxiov,
std::error_code& ec) const = 0;
virtual std::optional<std::span<uint8_t const>> header() const = 0;
virtual void set_num_workers(size_t num) = 0;
virtual void set_cache_tidy_config(cache_tidy_config const& cfg) = 0;

16
include/dwarfs/reader/internal/inode_reader_v2.h
View File

@ -66,14 +66,14 @@ class inode_reader_v2 {
size_t
readv(iovec_read_buf& buf, uint32_t inode, size_t size, file_off_t offset,
chunk_range chunks, std::error_code& ec) const {
return impl_->readv(buf, inode, size, offset, chunks, ec);
size_t maxiov, chunk_range chunks, std::error_code& ec) const {
return impl_->readv(buf, inode, size, offset, maxiov, chunks, ec);
}
std::vector<std::future<block_range>>
readv(uint32_t inode, size_t size, file_off_t offset, chunk_range chunks,
std::error_code& ec) const {
return impl_->readv(inode, size, offset, chunks, ec);
readv(uint32_t inode, size_t size, file_off_t offset, size_t maxiov,
chunk_range chunks, std::error_code& ec) const {
return impl_->readv(inode, size, offset, maxiov, chunks, ec);
}
void
@ -101,10 +101,10 @@ class inode_reader_v2 {
chunk_range chunks, std::error_code& ec) const = 0;
virtual size_t
readv(iovec_read_buf& buf, uint32_t inode, size_t size, file_off_t offset,
chunk_range chunks, std::error_code& ec) const = 0;
size_t maxiov, chunk_range chunks, std::error_code& ec) const = 0;
virtual std::vector<std::future<block_range>>
readv(uint32_t inode, size_t size, file_off_t offset, chunk_range chunks,
std::error_code& ec) const = 0;
readv(uint32_t inode, size_t size, file_off_t offset, size_t maxiov,
chunk_range chunks, std::error_code& ec) const = 0;
virtual void dump(std::ostream& os, const std::string& indent,
chunk_range chunks) const = 0;
virtual void set_num_workers(size_t num) = 0;

91
src/reader/filesystem_v2.cpp
View File

@ -62,6 +62,9 @@ using namespace dwarfs::internal;
namespace {
constexpr size_t const kDefaultMaxIOV{std::numeric_limits<size_t>::max()};
constexpr size_t const KReadFullFile{std::numeric_limits<size_t>::max()};
void check_section_logger(logger& lgr, fs_section const& section) {
LOG_PROXY(debug_logger_policy, lgr);
@ -259,6 +262,11 @@ class filesystem_ final : public filesystem_v2::impl {
file_off_t offset, std::error_code& ec) const override;
size_t readv(uint32_t inode, iovec_read_buf& buf, size_t size,
file_off_t offset) const override;
size_t
readv(uint32_t inode, iovec_read_buf& buf, size_t size, file_off_t offset,
size_t maxiov, std::error_code& ec) const override;
size_t readv(uint32_t inode, iovec_read_buf& buf, size_t size,
file_off_t offset, size_t maxiov) const override;
std::vector<std::future<block_range>> readv(uint32_t inode) const override;
std::vector<std::future<block_range>>
readv(uint32_t inode, std::error_code& ec) const override;
@ -267,6 +275,12 @@ class filesystem_ final : public filesystem_v2::impl {
std::vector<std::future<block_range>>
readv(uint32_t inode, size_t size, file_off_t offset,
std::error_code& ec) const override;
std::vector<std::future<block_range>>
readv(uint32_t inode, size_t size, file_off_t offset,
size_t maxiov) const override;
std::vector<std::future<block_range>>
readv(uint32_t inode, size_t size, file_off_t offset, size_t maxiov,
std::error_code& ec) const override;
std::optional<std::span<uint8_t const>> header() const override;
void set_num_workers(size_t num) override { ir_.set_num_workers(num); }
void set_cache_tidy_config(cache_tidy_config const& cfg) override {
@ -303,9 +317,9 @@ class filesystem_ final : public filesystem_v2::impl {
size_t read_ec(uint32_t inode, char* buf, size_t size, file_off_t offset,
std::error_code& ec) const;
size_t readv_ec(uint32_t inode, iovec_read_buf& buf, size_t size,
file_off_t offset, std::error_code& ec) const;
file_off_t offset, size_t maxiov, std::error_code& ec) const;
std::vector<std::future<block_range>>
readv_ec(uint32_t inode, size_t size, file_off_t offset,
readv_ec(uint32_t inode, size_t size, file_off_t offset, size_t maxiov,
std::error_code& ec) const;
LOG_PROXY_DECL(LoggerPolicy);
@ -859,14 +873,14 @@ template <typename LoggerPolicy>
std::string filesystem_<LoggerPolicy>::read_string(uint32_t inode,
std::error_code& ec) const {
PERFMON_CLS_SCOPED_SECTION(read_string_ec)
return read_string_ec(inode, std::numeric_limits<size_t>::max(), 0, ec);
return read_string_ec(inode, KReadFullFile, 0, ec);
}
template <typename LoggerPolicy>
std::string filesystem_<LoggerPolicy>::read_string(uint32_t inode) const {
PERFMON_CLS_SCOPED_SECTION(read_string)
return call_ec_throw([&](std::error_code& ec) {
return read_string_ec(inode, std::numeric_limits<size_t>::max(), 0, ec);
return read_string_ec(inode, KReadFullFile, 0, ec);
});
}
@ -916,12 +930,13 @@ size_t filesystem_<LoggerPolicy>::read(uint32_t inode, char* buf, size_t size,
}
template <typename LoggerPolicy>
size_t filesystem_<LoggerPolicy>::readv_ec(uint32_t inode, iovec_read_buf& buf,
size_t size, file_off_t offset,
std::error_code& ec) const {
size_t
filesystem_<LoggerPolicy>::readv_ec(uint32_t inode, iovec_read_buf& buf,
size_t size, file_off_t offset,
size_t maxiov, std::error_code& ec) const {
auto chunks = meta_.get_chunks(inode, ec);
if (!ec) {
return ir_.readv(buf, inode, size, offset, chunks, ec);
return ir_.readv(buf, inode, size, offset, maxiov, chunks, ec);
}
return 0;
}
@ -930,7 +945,7 @@ template <typename LoggerPolicy>
size_t filesystem_<LoggerPolicy>::readv(uint32_t inode, iovec_read_buf& buf,
std::error_code& ec) const {
PERFMON_CLS_SCOPED_SECTION(readv_iovec_ec)
return readv_ec(inode, buf, std::numeric_limits<size_t>::max(), 0, ec);
return readv_ec(inode, buf, KReadFullFile, 0, kDefaultMaxIOV, ec);
}
template <typename LoggerPolicy>
@ -938,7 +953,7 @@ size_t
filesystem_<LoggerPolicy>::readv(uint32_t inode, iovec_read_buf& buf) const {
PERFMON_CLS_SCOPED_SECTION(readv_iovec)
return call_ec_throw([&](std::error_code& ec) {
return readv_ec(inode, buf, std::numeric_limits<size_t>::max(), 0, ec);
return readv_ec(inode, buf, KReadFullFile, 0, kDefaultMaxIOV, ec);
});
}
@ -947,7 +962,7 @@ size_t filesystem_<LoggerPolicy>::readv(uint32_t inode, iovec_read_buf& buf,
size_t size, file_off_t offset,
std::error_code& ec) const {
PERFMON_CLS_SCOPED_SECTION(readv_iovec_ec)
return readv_ec(inode, buf, size, offset, ec);
return readv_ec(inode, buf, size, offset, kDefaultMaxIOV, ec);
}
template <typename LoggerPolicy>
@ -955,18 +970,37 @@ size_t filesystem_<LoggerPolicy>::readv(uint32_t inode, iovec_read_buf& buf,
size_t size, file_off_t offset) const {
PERFMON_CLS_SCOPED_SECTION(readv_iovec)
return call_ec_throw([&](std::error_code& ec) {
return readv_ec(inode, buf, size, offset, ec);
return readv_ec(inode, buf, size, offset, kDefaultMaxIOV, ec);
});
}
template <typename LoggerPolicy>
size_t
filesystem_<LoggerPolicy>::readv(uint32_t inode, iovec_read_buf& buf,
size_t size, file_off_t offset, size_t maxiov,
std::error_code& ec) const {
PERFMON_CLS_SCOPED_SECTION(readv_iovec_ec)
return readv_ec(inode, buf, size, offset, maxiov, ec);
}
template <typename LoggerPolicy>
size_t filesystem_<LoggerPolicy>::readv(uint32_t inode, iovec_read_buf& buf,
size_t size, file_off_t offset,
size_t maxiov) const {
PERFMON_CLS_SCOPED_SECTION(readv_iovec)
return call_ec_throw([&](std::error_code& ec) {
return readv_ec(inode, buf, size, offset, maxiov, ec);
});
}
template <typename LoggerPolicy>
std::vector<std::future<block_range>>
filesystem_<LoggerPolicy>::readv_ec(uint32_t inode, size_t size,
file_off_t offset,
file_off_t offset, size_t maxiov,
std::error_code& ec) const {
auto chunks = meta_.get_chunks(inode, ec);
if (!ec) {
return ir_.readv(inode, size, offset, chunks, ec);
return ir_.readv(inode, size, offset, maxiov, chunks, ec);
}
return {};
}
@ -975,7 +1009,7 @@ template <typename LoggerPolicy>
std::vector<std::future<block_range>>
filesystem_<LoggerPolicy>::readv(uint32_t inode, std::error_code& ec) const {
PERFMON_CLS_SCOPED_SECTION(readv_future_ec)
return readv_ec(inode, std::numeric_limits<size_t>::max(), 0, ec);
return readv_ec(inode, KReadFullFile, 0, kDefaultMaxIOV, ec);
}
template <typename LoggerPolicy>
@ -983,7 +1017,7 @@ std::vector<std::future<block_range>>
filesystem_<LoggerPolicy>::readv(uint32_t inode) const {
PERFMON_CLS_SCOPED_SECTION(readv_future)
return call_ec_throw([&](std::error_code& ec) {
return readv_ec(inode, std::numeric_limits<size_t>::max(), 0, ec);
return readv_ec(inode, KReadFullFile, 0, kDefaultMaxIOV, ec);
});
}
@ -992,7 +1026,7 @@ std::vector<std::future<block_range>>
filesystem_<LoggerPolicy>::readv(uint32_t inode, size_t size, file_off_t offset,
std::error_code& ec) const {
PERFMON_CLS_SCOPED_SECTION(readv_future_ec)
return readv_ec(inode, size, offset, ec);
return readv_ec(inode, size, offset, kDefaultMaxIOV, ec);
}
template <typename LoggerPolicy>
@ -1000,8 +1034,27 @@ std::vector<std::future<block_range>>
filesystem_<LoggerPolicy>::readv(uint32_t inode, size_t size,
file_off_t offset) const {
PERFMON_CLS_SCOPED_SECTION(readv_future)
return call_ec_throw(
[&](std::error_code& ec) { return readv_ec(inode, size, offset, ec); });
return call_ec_throw([&](std::error_code& ec) {
return readv_ec(inode, size, offset, kDefaultMaxIOV, ec);
});
}
template <typename LoggerPolicy>
std::vector<std::future<block_range>>
filesystem_<LoggerPolicy>::readv(uint32_t inode, size_t size, file_off_t offset,
size_t maxiov, std::error_code& ec) const {
PERFMON_CLS_SCOPED_SECTION(readv_future_ec)
return readv_ec(inode, size, offset, maxiov, ec);
}
template <typename LoggerPolicy>
std::vector<std::future<block_range>>
filesystem_<LoggerPolicy>::readv(uint32_t inode, size_t size, file_off_t offset,
size_t maxiov) const {
PERFMON_CLS_SCOPED_SECTION(readv_future)
return call_ec_throw([&](std::error_code& ec) {
return readv_ec(inode, size, offset, maxiov, ec);
});
}
template <typename LoggerPolicy>

40
src/reader/internal/inode_reader_v2.cpp
View File

@ -23,6 +23,7 @@
#include <cstdint>
#include <cstring>
#include <future>
#include <limits>
#include <mutex>
#include <ostream>
#include <utility>
@ -48,6 +49,8 @@ namespace dwarfs::reader::internal {
namespace {
constexpr size_t const kReadAllIOV{std::numeric_limits<size_t>::max()};
/**
* Offset cache configuration
*
@ -130,10 +133,10 @@ class inode_reader_ final : public inode_reader_v2::impl {
chunk_range chunks, std::error_code& ec) const override;
size_t
readv(iovec_read_buf& buf, uint32_t inode, size_t size, file_off_t offset,
chunk_range chunks, std::error_code& ec) const override;
size_t maxiov, chunk_range chunks, std::error_code& ec) const override;
std::vector<std::future<block_range>>
readv(uint32_t inode, size_t size, file_off_t offset, chunk_range chunks,
std::error_code& ec) const override;
readv(uint32_t inode, size_t size, file_off_t offset, size_t maxiov,
chunk_range chunks, std::error_code& ec) const override;
void dump(std::ostream& os, const std::string& indent,
chunk_range chunks) const override;
void set_num_workers(size_t num) override { cache_.set_num_workers(num); }
@ -151,12 +154,12 @@ class inode_reader_ final : public inode_reader_v2::impl {
using readahead_cache_type = folly::EvictingCacheMap<uint32_t, file_off_t>;
std::vector<std::future<block_range>>
read_internal(uint32_t inode, size_t size, file_off_t offset,
read_internal(uint32_t inode, size_t size, file_off_t offset, size_t maxiov,
chunk_range chunks, std::error_code& ec) const;
template <typename StoreFunc>
size_t read_internal(uint32_t inode, size_t size, file_off_t read_offset,
chunk_range chunks, std::error_code& ec,
size_t maxiov, chunk_range chunks, std::error_code& ec,
const StoreFunc& store) const;
void do_readahead(uint32_t inode, chunk_range::iterator it,
@ -237,6 +240,7 @@ template <typename LoggerPolicy>
std::vector<std::future<block_range>>
inode_reader_<LoggerPolicy>::read_internal(uint32_t inode, size_t const size,
file_off_t const read_offset,
size_t const maxiov,
chunk_range chunks,
std::error_code& ec) const {
std::vector<std::future<block_range>> ranges;
@ -315,7 +319,7 @@ inode_reader_<LoggerPolicy>::read_internal(uint32_t inode, size_t const size,
num_read += copysize;
if (num_read == size) {
if (num_read == size || ranges.size() >= maxiov) {
if (oc_ent) {
oc_ent->update(oc_upd, it_index, it_offset, chunksize);
offset_cache_.set(inode, std::move(oc_ent));
@ -340,13 +344,10 @@ inode_reader_<LoggerPolicy>::read_internal(uint32_t inode, size_t const size,
template <typename LoggerPolicy>
template <typename StoreFunc>
size_t
inode_reader_<LoggerPolicy>::read_internal(uint32_t inode, size_t size,
file_off_t offset,
chunk_range chunks,
std::error_code& ec,
const StoreFunc& store) const {
auto ranges = read_internal(inode, size, offset, chunks, ec);
size_t inode_reader_<LoggerPolicy>::read_internal(
uint32_t inode, size_t size, file_off_t offset, size_t const maxiov,
chunk_range chunks, std::error_code& ec, const StoreFunc& store) const {
auto ranges = read_internal(inode, size, offset, maxiov, chunks, ec);
if (ec) {
return 0;
@ -378,7 +379,7 @@ inode_reader_<LoggerPolicy>::read_string(uint32_t inode, size_t size,
PERFMON_CLS_SCOPED_SECTION(read_string)
PERFMON_SET_CONTEXT(static_cast<uint64_t>(offset), size);
auto ranges = read_internal(inode, size, offset, chunks, ec);
auto ranges = read_internal(inode, size, offset, kReadAllIOV, chunks, ec);
std::string res;
@ -411,7 +412,7 @@ size_t inode_reader_<LoggerPolicy>::read(char* buf, uint32_t inode, size_t size,
PERFMON_CLS_SCOPED_SECTION(read)
PERFMON_SET_CONTEXT(static_cast<uint64_t>(offset), size);
return read_internal(inode, size, offset, chunks, ec,
return read_internal(inode, size, offset, kReadAllIOV, chunks, ec,
[&](size_t num_read, const block_range& br) {
::memcpy(buf + num_read, br.data(), br.size());
});
@ -420,23 +421,24 @@ size_t inode_reader_<LoggerPolicy>::read(char* buf, uint32_t inode, size_t size,
template <typename LoggerPolicy>
std::vector<std::future<block_range>>
inode_reader_<LoggerPolicy>::readv(uint32_t inode, size_t const size,
file_off_t offset, chunk_range chunks,
file_off_t offset, size_t maxiov,
chunk_range chunks,
std::error_code& ec) const {
PERFMON_CLS_SCOPED_SECTION(readv_future)
PERFMON_SET_CONTEXT(static_cast<uint64_t>(offset), size);
return read_internal(inode, size, offset, chunks, ec);
return read_internal(inode, size, offset, maxiov, chunks, ec);
}
template <typename LoggerPolicy>
size_t inode_reader_<LoggerPolicy>::readv(iovec_read_buf& buf, uint32_t inode,
size_t size, file_off_t offset,
chunk_range chunks,
size_t maxiov, chunk_range chunks,
std::error_code& ec) const {
PERFMON_CLS_SCOPED_SECTION(readv_iovec)
PERFMON_SET_CONTEXT(static_cast<uint64_t>(offset), size);
auto rv = read_internal(inode, size, offset, chunks, ec,
auto rv = read_internal(inode, size, offset, maxiov, chunks, ec,
[&](size_t, const block_range& br) {
auto& iov = buf.buf.emplace_back();
iov.iov_base = const_cast<uint8_t*>(br.data());