TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-13 14:27:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

Compression metadata

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2023-07-22 17:59:23 +02:00
parent 3f0d7c14fd
commit c2da034983
7 changed files with 56 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
include/dwarfs/block_compressor.h
View File

@ -33,6 +33,8 @@
#include <utility>
#include <vector>
#include <folly/dynamic.h>
#include "dwarfs/compression.h"
namespace dwarfs {
@ -55,18 +57,24 @@ class block_compressor {
block_compressor(block_compressor&& bc) = default;
block_compressor& operator=(block_compressor&& rhs) = default;
std::vector<uint8_t> compress(std::vector<uint8_t> const& data) const {
return impl_->compress(data);
std::vector<uint8_t>
compress(std::vector<uint8_t> const& data, folly::dynamic meta) const {
return impl_->compress(data, std::move(meta));
}
std::vector<uint8_t> compress(std::vector<uint8_t>&& data) const {
return impl_->compress(std::move(data));
std::vector<uint8_t>
compress(std::vector<uint8_t>&& data, folly::dynamic meta) const {
return impl_->compress(std::move(data), std::move(meta));
}
compression_type type() const { return impl_->type(); }
std::string describe() const { return impl_->describe(); }
bool check_metadata(folly::dynamic meta) const {
return impl_->check_metadata(std::move(meta));
}
class impl {
public:
virtual ~impl() = default;
@ -74,12 +82,14 @@ class block_compressor {
virtual std::unique_ptr<impl> clone() const = 0;
virtual std::vector<uint8_t>
compress(const std::vector<uint8_t>& data) const = 0;
compress(const std::vector<uint8_t>& data, folly::dynamic meta) const = 0;
virtual std::vector<uint8_t>
compress(std::vector<uint8_t>&& data) const = 0;
compress(std::vector<uint8_t>&& data, folly::dynamic meta) const = 0;
virtual compression_type type() const = 0;
virtual std::string describe() const = 0;
virtual bool check_metadata(folly::dynamic meta) const = 0;
};
private:

11
src/dwarfs/compression/brotli.cpp
View File

@ -49,8 +49,8 @@ class brotli_block_compressor final : public block_compressor::impl {
return std::make_unique<brotli_block_compressor>(*this);
}
std::vector<uint8_t>
compress(const std::vector<uint8_t>& data) const override {
std::vector<uint8_t> compress(const std::vector<uint8_t>& data,
folly::dynamic /*meta*/) const override {
std::vector<uint8_t> compressed;
compressed.resize(folly::kMaxVarintLength64 +
::BrotliEncoderMaxCompressedSize(data.size()));
@ -69,8 +69,9 @@ class brotli_block_compressor final : public block_compressor::impl {
return compressed;
}
std::vector<uint8_t> compress(std::vector<uint8_t>&& data) const override {
return compress(data);
std::vector<uint8_t>
compress(std::vector<uint8_t>&& data, folly::dynamic meta) const override {
return compress(data, std::move(meta));
}
compression_type type() const override { return compression_type::BROTLI; }
@ -79,6 +80,8 @@ class brotli_block_compressor final : public block_compressor::impl {
return fmt::format("brotli [quality={}, lgwin={}]", quality_, window_bits_);
}
bool check_metadata(folly::dynamic /*meta*/) const override { return true; }
private:
uint32_t const quality_;
uint32_t const window_bits_;

11
src/dwarfs/compression/lz4.cpp
View File

@ -66,8 +66,8 @@ class lz4_block_compressor final : public block_compressor::impl {
return std::make_unique<lz4_block_compressor>(*this);
}
std::vector<uint8_t>
compress(const std::vector<uint8_t>& data) const override {
std::vector<uint8_t> compress(const std::vector<uint8_t>& data,
folly::dynamic /*meta*/) const override {
std::vector<uint8_t> compressed(
sizeof(uint32_t) + LZ4_compressBound(folly::to<int>(data.size())));
*reinterpret_cast<uint32_t*>(&compressed[0]) = data.size();
@ -84,14 +84,17 @@ class lz4_block_compressor final : public block_compressor::impl {
return compressed;
}
std::vector<uint8_t> compress(std::vector<uint8_t>&& data) const override {
return compress(data);
std::vector<uint8_t>
compress(std::vector<uint8_t>&& data, folly::dynamic meta) const override {
return compress(data, std::move(meta));
}
compression_type type() const override { return compression_type::LZ4; }
std::string describe() const override { return Policy::describe(level_); }
bool check_metadata(folly::dynamic /*meta*/) const override { return true; }
private:
const int level_;
};

12
src/dwarfs/compression/lzma.cpp
View File

@ -63,16 +63,19 @@ class lzma_block_compressor final : public block_compressor::impl {
return std::make_unique<lzma_block_compressor>(*this);
}
std::vector<uint8_t> compress(const std::vector<uint8_t>& data,
folly::dynamic meta) const override;
std::vector<uint8_t>
compress(const std::vector<uint8_t>& data) const override;
std::vector<uint8_t> compress(std::vector<uint8_t>&& data) const override {
return compress(data);
compress(std::vector<uint8_t>&& data, folly::dynamic meta) const override {
return compress(data, std::move(meta));
}
compression_type type() const override { return compression_type::LZMA; }
std::string describe() const override { return description_; }
bool check_metadata(folly::dynamic /*meta*/) const override { return true; }
private:
std::vector<uint8_t>
compress(const std::vector<uint8_t>& data, const lzma_filter* filters) const;
@ -174,7 +177,8 @@ lzma_block_compressor::compress(const std::vector<uint8_t>& data,
}
std::vector<uint8_t>
lzma_block_compressor::compress(const std::vector<uint8_t>& data) const {
lzma_block_compressor::compress(const std::vector<uint8_t>& data,
folly::dynamic /*meta*/) const {
std::vector<uint8_t> best = compress(data, &filters_[1]);
if (filters_[0].id != LZMA_VLI_UNKNOWN) {

9
src/dwarfs/compression/null.cpp
View File

@ -37,18 +37,21 @@ class null_block_compressor final : public block_compressor::impl {
return std::make_unique<null_block_compressor>(*this);
}
std::vector<uint8_t>
compress(const std::vector<uint8_t>& data) const override {
std::vector<uint8_t> compress(const std::vector<uint8_t>& data,
folly::dynamic /*meta*/) const override {
return data;
}
std::vector<uint8_t> compress(std::vector<uint8_t>&& data) const override {
std::vector<uint8_t> compress(std::vector<uint8_t>&& data,
folly::dynamic /*meta*/) const override {
return std::move(data);
}
compression_type type() const override { return compression_type::NONE; }
std::string describe() const override { return "null"; }
bool check_metadata(folly::dynamic /*meta*/) const override { return true; }
};
class null_block_decompressor final : public block_decompressor::impl {

14
src/dwarfs/compression/zstd.cpp
View File

@ -54,11 +54,12 @@ class zstd_block_compressor final : public block_compressor::impl {
return std::make_unique<zstd_block_compressor>(*this);
}
std::vector<uint8_t>
compress(const std::vector<uint8_t>& data) const override;
std::vector<uint8_t> compress(const std::vector<uint8_t>& data,
folly::dynamic meta) const override;
std::vector<uint8_t> compress(std::vector<uint8_t>&& data) const override {
return compress(data);
std::vector<uint8_t>
compress(std::vector<uint8_t>&& data, folly::dynamic meta) const override {
return compress(data, std::move(meta));
}
compression_type type() const override { return compression_type::ZSTD; }
@ -67,6 +68,8 @@ class zstd_block_compressor final : public block_compressor::impl {
return fmt::format("zstd [level={}]", level_);
}
bool check_metadata(folly::dynamic /*meta*/) const override { return true; }
private:
class scoped_context;
@ -143,7 +146,8 @@ std::weak_ptr<zstd_block_compressor::context_manager>
zstd_block_compressor::s_ctxmgr;
std::vector<uint8_t>
zstd_block_compressor::compress(const std::vector<uint8_t>& data) const {
zstd_block_compressor::compress(const std::vector<uint8_t>& data,
folly::dynamic /*meta*/) const {
std::vector<uint8_t> compressed(ZSTD_compressBound(data.size()));
scoped_context ctx(*ctxmgr_);
auto size = ZSTD_compressCCtx(ctx.get(), compressed.data(), compressed.size(),

4
src/dwarfs/filesystem_writer.cpp
View File

@ -101,7 +101,9 @@ class raw_fsblock : public fsblock::impl {
wg.add_job([this, prom = std::move(prom)]() mutable {
try {
auto tmp = std::make_shared<block_data>(bc_.compress(data_->vec()));
// TODO: metadata
auto tmp = std::make_shared<block_data>(
bc_.compress(data_->vec(), folly::dynamic()));
{
std::lock_guard lock(mx_);