TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-14 06:48:39 -04:00
Code Issues Packages Projects Releases Wiki Activity

Compression metadata

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2023-07-22 17:59:23 +02:00
parent 3f0d7c14fd
commit c2da034983
7 changed files with 56 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
include/dwarfs/block_compressor.h
View File

@ -33,6 +33,8 @@
#include <utility> #include <utility>
#include <vector> #include <vector>
#include <folly/dynamic.h>
#include "dwarfs/compression.h" #include "dwarfs/compression.h"
namespace dwarfs { namespace dwarfs {
@ -55,18 +57,24 @@ class block_compressor {
block_compressor(block_compressor&& bc) = default; block_compressor(block_compressor&& bc) = default;
block_compressor& operator=(block_compressor&& rhs) = default; block_compressor& operator=(block_compressor&& rhs) = default;
std::vector<uint8_t> compress(std::vector<uint8_t> const& data) const { std::vector<uint8_t>
return impl_->compress(data); compress(std::vector<uint8_t> const& data, folly::dynamic meta) const {
return impl_->compress(data, std::move(meta));
} }
std::vector<uint8_t> compress(std::vector<uint8_t>&& data) const { std::vector<uint8_t>
return impl_->compress(std::move(data)); compress(std::vector<uint8_t>&& data, folly::dynamic meta) const {
return impl_->compress(std::move(data), std::move(meta));
} }
compression_type type() const { return impl_->type(); } compression_type type() const { return impl_->type(); }
std::string describe() const { return impl_->describe(); } std::string describe() const { return impl_->describe(); }
bool check_metadata(folly::dynamic meta) const {
return impl_->check_metadata(std::move(meta));
}
class impl { class impl {
public: public:
virtual ~impl() = default; virtual ~impl() = default;
@ -74,12 +82,14 @@ class block_compressor {
virtual std::unique_ptr<impl> clone() const = 0; virtual std::unique_ptr<impl> clone() const = 0;
virtual std::vector<uint8_t> virtual std::vector<uint8_t>
compress(const std::vector<uint8_t>& data) const = 0; compress(const std::vector<uint8_t>& data, folly::dynamic meta) const = 0;
virtual std::vector<uint8_t> virtual std::vector<uint8_t>
compress(std::vector<uint8_t>&& data) const = 0; compress(std::vector<uint8_t>&& data, folly::dynamic meta) const = 0;
virtual compression_type type() const = 0; virtual compression_type type() const = 0;
virtual std::string describe() const = 0; virtual std::string describe() const = 0;
virtual bool check_metadata(folly::dynamic meta) const = 0;
}; };
private: private:

11
src/dwarfs/compression/brotli.cpp
View File

@ -49,8 +49,8 @@ class brotli_block_compressor final : public block_compressor::impl {
return std::make_unique<brotli_block_compressor>(*this); return std::make_unique<brotli_block_compressor>(*this);
} }
std::vector<uint8_t> std::vector<uint8_t> compress(const std::vector<uint8_t>& data,
compress(const std::vector<uint8_t>& data) const override { folly::dynamic /*meta*/) const override {
std::vector<uint8_t> compressed; std::vector<uint8_t> compressed;
compressed.resize(folly::kMaxVarintLength64 + compressed.resize(folly::kMaxVarintLength64 +
::BrotliEncoderMaxCompressedSize(data.size())); ::BrotliEncoderMaxCompressedSize(data.size()));
@ -69,8 +69,9 @@ class brotli_block_compressor final : public block_compressor::impl {
return compressed; return compressed;
} }
std::vector<uint8_t> compress(std::vector<uint8_t>&& data) const override { std::vector<uint8_t>
return compress(data); compress(std::vector<uint8_t>&& data, folly::dynamic meta) const override {
return compress(data, std::move(meta));
} }
compression_type type() const override { return compression_type::BROTLI; } compression_type type() const override { return compression_type::BROTLI; }
@ -79,6 +80,8 @@ class brotli_block_compressor final : public block_compressor::impl {
return fmt::format("brotli [quality={}, lgwin={}]", quality_, window_bits_); return fmt::format("brotli [quality={}, lgwin={}]", quality_, window_bits_);
} }
bool check_metadata(folly::dynamic /*meta*/) const override { return true; }
private: private:
uint32_t const quality_; uint32_t const quality_;
uint32_t const window_bits_; uint32_t const window_bits_;

11
src/dwarfs/compression/lz4.cpp
View File

@ -66,8 +66,8 @@ class lz4_block_compressor final : public block_compressor::impl {
return std::make_unique<lz4_block_compressor>(*this); return std::make_unique<lz4_block_compressor>(*this);
} }
std::vector<uint8_t> std::vector<uint8_t> compress(const std::vector<uint8_t>& data,
compress(const std::vector<uint8_t>& data) const override { folly::dynamic /*meta*/) const override {
std::vector<uint8_t> compressed( std::vector<uint8_t> compressed(
sizeof(uint32_t) + LZ4_compressBound(folly::to<int>(data.size()))); sizeof(uint32_t) + LZ4_compressBound(folly::to<int>(data.size())));
*reinterpret_cast<uint32_t*>(&compressed[0]) = data.size(); *reinterpret_cast<uint32_t*>(&compressed[0]) = data.size();
@ -84,14 +84,17 @@ class lz4_block_compressor final : public block_compressor::impl {
return compressed; return compressed;
} }
std::vector<uint8_t> compress(std::vector<uint8_t>&& data) const override { std::vector<uint8_t>
return compress(data); compress(std::vector<uint8_t>&& data, folly::dynamic meta) const override {
return compress(data, std::move(meta));
} }
compression_type type() const override { return compression_type::LZ4; } compression_type type() const override { return compression_type::LZ4; }
std::string describe() const override { return Policy::describe(level_); } std::string describe() const override { return Policy::describe(level_); }
bool check_metadata(folly::dynamic /*meta*/) const override { return true; }
private: private:
const int level_; const int level_;
}; };

12
src/dwarfs/compression/lzma.cpp
View File

@ -63,16 +63,19 @@ class lzma_block_compressor final : public block_compressor::impl {
return std::make_unique<lzma_block_compressor>(*this); return std::make_unique<lzma_block_compressor>(*this);
} }
std::vector<uint8_t> compress(const std::vector<uint8_t>& data,
folly::dynamic meta) const override;
std::vector<uint8_t> std::vector<uint8_t>
compress(const std::vector<uint8_t>& data) const override; compress(std::vector<uint8_t>&& data, folly::dynamic meta) const override {
std::vector<uint8_t> compress(std::vector<uint8_t>&& data) const override { return compress(data, std::move(meta));
return compress(data);
} }
compression_type type() const override { return compression_type::LZMA; } compression_type type() const override { return compression_type::LZMA; }
std::string describe() const override { return description_; } std::string describe() const override { return description_; }
bool check_metadata(folly::dynamic /*meta*/) const override { return true; }
private: private:
std::vector<uint8_t> std::vector<uint8_t>
compress(const std::vector<uint8_t>& data, const lzma_filter* filters) const; compress(const std::vector<uint8_t>& data, const lzma_filter* filters) const;
@ -174,7 +177,8 @@ lzma_block_compressor::compress(const std::vector<uint8_t>& data,
} }
std::vector<uint8_t> std::vector<uint8_t>
lzma_block_compressor::compress(const std::vector<uint8_t>& data) const { lzma_block_compressor::compress(const std::vector<uint8_t>& data,
folly::dynamic /*meta*/) const {
std::vector<uint8_t> best = compress(data, &filters_[1]); std::vector<uint8_t> best = compress(data, &filters_[1]);
if (filters_[0].id != LZMA_VLI_UNKNOWN) { if (filters_[0].id != LZMA_VLI_UNKNOWN) {

9
src/dwarfs/compression/null.cpp
View File

@ -37,18 +37,21 @@ class null_block_compressor final : public block_compressor::impl {
return std::make_unique<null_block_compressor>(*this); return std::make_unique<null_block_compressor>(*this);
} }
std::vector<uint8_t> std::vector<uint8_t> compress(const std::vector<uint8_t>& data,
compress(const std::vector<uint8_t>& data) const override { folly::dynamic /*meta*/) const override {
return data; return data;
} }
std::vector<uint8_t> compress(std::vector<uint8_t>&& data) const override { std::vector<uint8_t> compress(std::vector<uint8_t>&& data,
folly::dynamic /*meta*/) const override {
return std::move(data); return std::move(data);
} }
compression_type type() const override { return compression_type::NONE; } compression_type type() const override { return compression_type::NONE; }
std::string describe() const override { return "null"; } std::string describe() const override { return "null"; }
bool check_metadata(folly::dynamic /*meta*/) const override { return true; }
}; };
class null_block_decompressor final : public block_decompressor::impl { class null_block_decompressor final : public block_decompressor::impl {

14
src/dwarfs/compression/zstd.cpp
View File

@ -54,11 +54,12 @@ class zstd_block_compressor final : public block_compressor::impl {
return std::make_unique<zstd_block_compressor>(*this); return std::make_unique<zstd_block_compressor>(*this);
} }
std::vector<uint8_t> std::vector<uint8_t> compress(const std::vector<uint8_t>& data,
compress(const std::vector<uint8_t>& data) const override; folly::dynamic meta) const override;
std::vector<uint8_t> compress(std::vector<uint8_t>&& data) const override { std::vector<uint8_t>
return compress(data); compress(std::vector<uint8_t>&& data, folly::dynamic meta) const override {
return compress(data, std::move(meta));
} }
compression_type type() const override { return compression_type::ZSTD; } compression_type type() const override { return compression_type::ZSTD; }
@ -67,6 +68,8 @@ class zstd_block_compressor final : public block_compressor::impl {
return fmt::format("zstd [level={}]", level_); return fmt::format("zstd [level={}]", level_);
} }
bool check_metadata(folly::dynamic /*meta*/) const override { return true; }
private: private:
class scoped_context; class scoped_context;
@ -143,7 +146,8 @@ std::weak_ptr<zstd_block_compressor::context_manager>
zstd_block_compressor::s_ctxmgr; zstd_block_compressor::s_ctxmgr;
std::vector<uint8_t> std::vector<uint8_t>
zstd_block_compressor::compress(const std::vector<uint8_t>& data) const { zstd_block_compressor::compress(const std::vector<uint8_t>& data,
folly::dynamic /*meta*/) const {
std::vector<uint8_t> compressed(ZSTD_compressBound(data.size())); std::vector<uint8_t> compressed(ZSTD_compressBound(data.size()));
scoped_context ctx(*ctxmgr_); scoped_context ctx(*ctxmgr_);
auto size = ZSTD_compressCCtx(ctx.get(), compressed.data(), compressed.size(), auto size = ZSTD_compressCCtx(ctx.get(), compressed.data(), compressed.size(),

4
src/dwarfs/filesystem_writer.cpp
View File

@ -101,7 +101,9 @@ class raw_fsblock : public fsblock::impl {
wg.add_job([this, prom = std::move(prom)]() mutable { wg.add_job([this, prom = std::move(prom)]() mutable {
try { try {
auto tmp = std::make_shared<block_data>(bc_.compress(data_->vec())); // TODO: metadata
auto tmp = std::make_shared<block_data>(
bc_.compress(data_->vec(), folly::dynamic()));
{ {
std::lock_guard lock(mx_); std::lock_guard lock(mx_);