TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-09 04:19:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

refactor(segmenter): use byte_buffer, but still needs raw_vector

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2025-03-30 13:12:17 +02:00
parent f1ae86ab2f
commit 6541afbe06
2 changed files with 31 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/dwarfs/byte_buffer.h
View File

@ -67,6 +67,8 @@ class mutable_byte_buffer_interface : public byte_buffer_interface {
class shared_byte_buffer { class shared_byte_buffer {
public: public:
using value_type = uint8_t;
shared_byte_buffer() = default; shared_byte_buffer() = default;
explicit shared_byte_buffer(std::shared_ptr<byte_buffer_interface const> bb) explicit shared_byte_buffer(std::shared_ptr<byte_buffer_interface const> bb)
@ -114,6 +116,8 @@ class shared_byte_buffer {
class mutable_byte_buffer { class mutable_byte_buffer {
public: public:
using value_type = uint8_t;
explicit mutable_byte_buffer( explicit mutable_byte_buffer(
std::shared_ptr<mutable_byte_buffer_interface> bb) std::shared_ptr<mutable_byte_buffer_interface> bb)
: bb_{std::move(bb)} {} : bb_{std::move(bb)} {}

38
src/writer/segmenter.cpp
View File

@ -507,11 +507,13 @@ using MultiBlockSegmentationPolicy =
BasicSegmentationPolicy<GranularityPolicy, true, true>; BasicSegmentationPolicy<GranularityPolicy, true, true>;
template <typename T, typename GranularityPolicy> template <typename T, typename GranularityPolicy>
class granular_vector_adapter : private GranularityPolicy { class basic_granular_container_adapter : private GranularityPolicy {
public: public:
using value_type = typename T::value_type;
template <typename... PolicyArgs> template <typename... PolicyArgs>
DWARFS_FORCE_INLINE DWARFS_FORCE_INLINE
granular_vector_adapter(std::vector<T>& v, PolicyArgs&&... args) basic_granular_container_adapter(T& v, PolicyArgs&&... args)
: GranularityPolicy(std::forward<PolicyArgs>(args)...) : GranularityPolicy(std::forward<PolicyArgs>(args)...)
, v_{v} {} , v_{v} {}
@ -519,8 +521,8 @@ class granular_vector_adapter : private GranularityPolicy {
return this->bytes_to_frames(v_.size()); return this->bytes_to_frames(v_.size());
} }
DWARFS_FORCE_INLINE void DWARFS_FORCE_INLINE void append(
append(granular_span_adapter<T const, GranularityPolicy> const& span) { granular_span_adapter<value_type const, GranularityPolicy> const& span) {
auto raw = span.raw(); auto raw = span.raw();
auto off = v_.size(); auto off = v_.size();
v_.resize(off + raw.size()); v_.resize(off + raw.size());
@ -529,7 +531,8 @@ class granular_vector_adapter : private GranularityPolicy {
DWARFS_FORCE_INLINE int DWARFS_FORCE_INLINE int
compare(size_t offset, compare(size_t offset,
granular_span_adapter<T const, GranularityPolicy> const& span) const { granular_span_adapter<value_type const, GranularityPolicy> const&
span) const {
auto raw = span.raw(); auto raw = span.raw();
return std::memcmp(v_.data() + this->frames_to_bytes(offset), raw.data(), return std::memcmp(v_.data() + this->frames_to_bytes(offset), raw.data(),
raw.size()); raw.size());
@ -537,22 +540,31 @@ class granular_vector_adapter : private GranularityPolicy {
template <typename H> template <typename H>
DWARFS_FORCE_INLINE void update_hash(H& hasher, size_t offset) const { DWARFS_FORCE_INLINE void update_hash(H& hasher, size_t offset) const {
offset = this->frames_to_bytes(offset); auto p = v_.data() + this->frames_to_bytes(offset);
this->for_bytes_in_frame([&] { hasher.update(v_[offset++]); }); this->for_bytes_in_frame([&] { hasher.update(*p++); });
} }
template <typename H> template <typename H>
DWARFS_FORCE_INLINE void DWARFS_FORCE_INLINE void
update_hash(H& hasher, size_t from, size_t to) const { update_hash(H& hasher, size_t from, size_t to) const {
from = this->frames_to_bytes(from); auto const p = v_.data();
to = this->frames_to_bytes(to); auto pfrom = p + this->frames_to_bytes(from);
this->for_bytes_in_frame([&] { hasher.update(v_[from++], v_[to++]); }); auto pto = p + this->frames_to_bytes(to);
this->for_bytes_in_frame([&] { hasher.update(*pfrom++, *pto++); });
} }
private: private:
std::vector<T>& v_; T& v_;
}; };
template <typename T, typename GranularityPolicy>
using granular_vector_adapter =
basic_granular_container_adapter<std::vector<T>, GranularityPolicy>;
template <typename GranularityPolicy>
using granular_byte_buffer_adapter =
basic_granular_container_adapter<mutable_byte_buffer, GranularityPolicy>;
template <typename LoggerPolicy, typename GranularityPolicy> template <typename LoggerPolicy, typename GranularityPolicy>
class active_block : private GranularityPolicy { class active_block : private GranularityPolicy {
private: private:
@ -883,6 +895,10 @@ active_block<LoggerPolicy, GranularityPolicy>::append_bytes(
auto v = this->template create< auto v = this->template create<
granular_vector_adapter<uint8_t, GranularityPolicy>>(data_.raw_vector()); granular_vector_adapter<uint8_t, GranularityPolicy>>(data_.raw_vector());
// TODO: this works in theory, but slows down the segmenter by almost 10%
// auto v = this->template create<
// granular_byte_buffer_adapter<GranularityPolicy>>(data_);
auto offset = v.size(); auto offset = v.size();
DWARFS_CHECK(offset + src.size() <= capacity_in_frames_, DWARFS_CHECK(offset + src.size() <= capacity_in_frames_,