feat(metadata_freezer): log metadata freezing time

This commit is contained in:
Marcus Holland-Moritz 2024-11-25 18:00:02 +01:00
parent b1f6fd72a7
commit 886a107a0e
3 changed files with 49 additions and 7 deletions

View File

@ -24,12 +24,15 @@
#pragma once
#include <cstdint>
#include <memory>
#include <utility>
#include <dwarfs/byte_buffer.h>
namespace dwarfs {
class logger;
namespace thrift::metadata {
class metadata;
}
@ -38,8 +41,24 @@ namespace writer::internal {
class metadata_freezer {
public:
static std::pair<shared_byte_buffer, shared_byte_buffer>
freeze(thrift::metadata::metadata const& data);
metadata_freezer(logger& lgr);
~metadata_freezer();
std::pair<shared_byte_buffer, shared_byte_buffer>
freeze(thrift::metadata::metadata const& data) const {
return impl_->freeze(data);
}
class impl {
public:
virtual ~impl() = default;
virtual std::pair<shared_byte_buffer, shared_byte_buffer>
freeze(thrift::metadata::metadata const& data) const = 0;
};
private:
std::unique_ptr<impl> impl_;
};
} // namespace writer::internal

View File

@ -24,6 +24,7 @@
#include <thrift/lib/cpp2/frozen/FrozenUtil.h>
#include <thrift/lib/cpp2/protocol/Serializer.h>
#include <dwarfs/logger.h>
#include <dwarfs/malloc_byte_buffer.h>
#include <dwarfs/writer/internal/metadata_freezer.h>
@ -60,11 +61,33 @@ std::pair<shared_byte_buffer, shared_byte_buffer> freeze_to_buffer(T const& x) {
return {schema_buffer.share(), data_buffer.share()};
}
template <typename LoggerPolicy>
class metadata_freezer_ : public metadata_freezer::impl {
public:
explicit metadata_freezer_(logger& lgr)
: LOG_PROXY_INIT(lgr) {}
std::pair<shared_byte_buffer, shared_byte_buffer>
freeze(thrift::metadata::metadata const& data) const override {
auto ti = LOG_TIMED_VERBOSE;
auto rv = freeze_to_buffer(data);
ti << "freezing metadata to " << rv.second.size() << " bytes...";
return rv;
}
private:
LOG_PROXY_DECL(LoggerPolicy);
};
} // namespace
std::pair<shared_byte_buffer, shared_byte_buffer>
metadata_freezer::freeze(thrift::metadata::metadata const& data) {
return freeze_to_buffer(data);
}
metadata_freezer::metadata_freezer(logger& lgr)
: impl_{
make_unique_logging_object<impl, metadata_freezer_, logger_policies>(
lgr)} {}
metadata_freezer::~metadata_freezer() = default;
} // namespace dwarfs::writer::internal

View File

@ -949,7 +949,7 @@ void scanner_<LoggerPolicy>::scan(
mdb.set_total_hardlink_size(prog.hardlink_size);
mdb.gather_global_entry_data(ge_data);
auto [schema, data] = metadata_freezer::freeze(mdb.build());
auto [schema, data] = metadata_freezer(LOG_GET_LOGGER).freeze(mdb.build());
LOG_VERBOSE << "uncompressed metadata size: " << size_with_unit(data.size());