TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-15 07:16:13 -04:00
Code Issues Packages Projects Releases Wiki Activity

chore(segmenter): add debug logging with memory usage information

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2025-05-22 13:54:17 +02:00
parent 6240c601c7
commit b587042215
1 changed files with 31 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/writer/segmenter.cpp
View File

@ -149,16 +149,28 @@ class fast_multimap {
return false; return false;
} }
void clear() {
values_.clear();
collisions_.clear();
}
DWARFS_FORCE_INLINE blockhash_t const& values() const { return values_; }; DWARFS_FORCE_INLINE blockhash_t const& values() const { return values_; };
DWARFS_FORCE_INLINE collision_t const& collisions() const { DWARFS_FORCE_INLINE collision_t const& collisions() const {
return collisions_; return collisions_;
}; };
size_t memory_usage() const {
auto phmap_mem = []<typename T>(T const& m) {
return m.capacity() * sizeof(typename T::slot_type) + m.capacity() + 1;
};
auto mem = phmap_mem(values_) + phmap_mem(collisions_);
for (auto const& c : collisions_) {
if (c.second.size() > MaxCollInline) {
mem +=
c.second.capacity() * sizeof(typename collision_vector::value_type);
}
}
return mem;
}
private: private:
blockhash_t values_; blockhash_t values_;
collision_t collisions_; collision_t collisions_;
@ -245,6 +257,8 @@ class alignas(64) bloom_filter {
std::transform(cbegin(), cend(), other.cbegin(), begin(), std::bit_or<>{}); std::transform(cbegin(), cend(), other.cbegin(), begin(), std::bit_or<>{});
} }
size_t memory_usage() const { return size_ / 8; }
private: private:
DWARFS_FORCE_INLINE bits_type const* cbegin() const { return bits_; } DWARFS_FORCE_INLINE bits_type const* cbegin() const { return bits_; }
DWARFS_FORCE_INLINE bits_type const* cend() const { DWARFS_FORCE_INLINE bits_type const* cend() const {
@ -592,6 +606,18 @@ class active_block : private GranularityPolicy {
data_.reserve(this->frames_to_bytes(capacity_in_frames_)); data_.reserve(this->frames_to_bytes(capacity_in_frames_));
} }
~active_block() {
LOG_DEBUG << "block " << num_ << " destroyed, "
<< size_with_unit(offsets_.memory_usage()) << " offset memory ("
<< fmt::format("{:.1f}",
static_cast<double>(offsets_.memory_usage()) /
offsets_.values().size())
<< " bytes per offset, " << offsets_.values().size()
<< " offsets, " << offsets_.collisions().size()
<< " collisions), " << size_with_unit(filter_.memory_usage())
<< " bloom filter";
}
DWARFS_FORCE_INLINE size_t num() const { return num_; } DWARFS_FORCE_INLINE size_t num() const { return num_; }
DWARFS_FORCE_INLINE size_t size_in_frames() const { DWARFS_FORCE_INLINE size_t size_in_frames() const {