From 99967481347ee2f2064e37e5afa26c3fb94982cf Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Wed, 9 Apr 2025 13:06:48 +0200 Subject: [PATCH] feat: replace vector_byte_buffer with malloc_byte_buffer The latter doesn't always explicitly initialize the allocated memory, which can help improve performance. --- cmake/libdwarfs.cmake | 4 +- include/dwarfs/byte_buffer.h | 12 ++- include/dwarfs/internal/malloc_buffer.h | 93 +++++++++++++++++++ ...tor_byte_buffer.h => malloc_byte_buffer.h} | 9 +- ...factory.h => malloc_byte_buffer_factory.h} | 2 +- src/block_decompressor.cpp | 4 +- src/compression/brotli.cpp | 4 +- src/compression/flac.cpp | 4 +- src/compression/lz4.cpp | 4 +- src/compression/lzma.cpp | 4 +- src/compression/ricepp.cpp | 4 +- src/compression/zstd.cpp | 4 +- src/history.cpp | 4 +- src/internal/malloc_buffer.cpp | 85 +++++++++++++++++ ...byte_buffer.cpp => malloc_byte_buffer.cpp} | 56 ++++++----- ...ory.cpp => malloc_byte_buffer_factory.cpp} | 12 +-- .../block_cache_byte_buffer_factory.cpp | 8 +- src/reader/filesystem_v2.cpp | 2 +- src/writer/filesystem_writer.cpp | 4 +- src/writer/internal/metadata_freezer.cpp | 6 +- src/writer/segmenter.cpp | 23 ++--- test/flac_compressor_test.cpp | 4 +- test/ricepp_compressor_test.cpp | 4 +- 23 files changed, 274 insertions(+), 82 deletions(-) create mode 100644 include/dwarfs/internal/malloc_buffer.h rename include/dwarfs/{vector_byte_buffer.h => malloc_byte_buffer.h} (89%) rename include/dwarfs/{vector_byte_buffer_factory.h => malloc_byte_buffer_factory.h} (97%) create mode 100644 src/internal/malloc_buffer.cpp rename src/{vector_byte_buffer.cpp => malloc_byte_buffer.cpp} (66%) rename src/{vector_byte_buffer_factory.cpp => malloc_byte_buffer_factory.cpp} (82%) diff --git a/cmake/libdwarfs.cmake b/cmake/libdwarfs.cmake index 52559dac..65df5fc9 100644 --- a/cmake/libdwarfs.cmake +++ b/cmake/libdwarfs.cmake @@ -34,6 +34,7 @@ add_library( src/history.cpp src/library_dependencies.cpp src/logger.cpp + src/malloc_byte_buffer.cpp src/mapped_byte_buffer.cpp src/mmap.cpp src/option_map.cpp @@ -44,8 +45,6 @@ add_library( src/thread_pool.cpp src/util.cpp src/varint.cpp - src/vector_byte_buffer.cpp - src/vector_byte_buffer_factory.cpp src/xattr.cpp src/internal/features.cpp @@ -53,6 +52,7 @@ add_library( src/internal/fs_section.cpp src/internal/fs_section_checker.cpp src/internal/glob_to_regex.cpp + src/internal/malloc_buffer.cpp src/internal/metadata_utils.cpp src/internal/string_table.cpp src/internal/unicode_case_folding.cpp diff --git a/include/dwarfs/byte_buffer.h b/include/dwarfs/byte_buffer.h index 50029432..e335e314 100644 --- a/include/dwarfs/byte_buffer.h +++ b/include/dwarfs/byte_buffer.h @@ -49,6 +49,12 @@ compare_spans(std::span lhs, std::span rhs); } // namespace detail +namespace internal { + +class malloc_buffer; + +} + class byte_buffer_interface { public: virtual ~byte_buffer_interface() = default; @@ -72,9 +78,7 @@ class mutable_byte_buffer_interface : public byte_buffer_interface { // that would reallocate the buffer will throw. virtual void freeze_location() = 0; - // TODO: See if we can do without this. This will *only* be implemented - // in the vector_byte_buffer, other implementations will throw. - virtual std::vector& raw_vector() = 0; + virtual internal::malloc_buffer& raw_buffer() = 0; }; class shared_byte_buffer { @@ -164,7 +168,7 @@ class mutable_byte_buffer { void freeze_location() { bb_->freeze_location(); } - std::vector& raw_vector() { return bb_->raw_vector(); } + internal::malloc_buffer& raw_buffer() { return bb_->raw_buffer(); } void swap(mutable_byte_buffer& other) noexcept { std::swap(bb_, other.bb_); } diff --git a/include/dwarfs/internal/malloc_buffer.h b/include/dwarfs/internal/malloc_buffer.h new file mode 100644 index 00000000..84dc806e --- /dev/null +++ b/include/dwarfs/internal/malloc_buffer.h @@ -0,0 +1,93 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the “Software”), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * SPDX-License-Identifier: MIT + */ + +#pragma once + +#include +#include +#include + +namespace dwarfs::internal { + +class malloc_buffer { + public: + using value_type = uint8_t; + + malloc_buffer() = default; + malloc_buffer(size_t size); + malloc_buffer(void const* data, size_t size); + malloc_buffer(std::span data); + + ~malloc_buffer(); + + malloc_buffer(malloc_buffer&&) = default; + malloc_buffer& operator=(malloc_buffer&&) = default; + + bool empty() const { return size_ == 0; } + + size_t size() const { return size_; } + size_t capacity() const { return capacity_; } + + value_type const* data() const { return data_; } + value_type* data() { return data_; } + + void append(void const* data, size_t size) { + reserve(size_ + size); + copy(data_ + size_, data, size); + size_ += size; + } + + void clear() { size_ = 0; } + + void resize(size_t new_size) { + reserve(new_size); + size_ = new_size; + } + + void reserve(size_t new_capacity) { + if (new_capacity > capacity_) { + grow(new_capacity); + } + } + + void shrink_to_fit(); + + private: + void grow(size_t new_size); + void resize_buffer(size_t new_size); + static void copy(void* dest, void const* src, size_t size) { + // TODO: try std::copy or even something custom + std::memcpy(dest, src, size); + } + + value_type* data_{nullptr}; + size_t size_{0}; + size_t capacity_{0}; +}; + +} // namespace dwarfs::internal diff --git a/include/dwarfs/vector_byte_buffer.h b/include/dwarfs/malloc_byte_buffer.h similarity index 89% rename from include/dwarfs/vector_byte_buffer.h rename to include/dwarfs/malloc_byte_buffer.h index cb1e0dd5..0f420558 100644 --- a/include/dwarfs/vector_byte_buffer.h +++ b/include/dwarfs/malloc_byte_buffer.h @@ -34,14 +34,19 @@ namespace dwarfs { -class vector_byte_buffer { +namespace internal { +class malloc_buffer; +} + +class malloc_byte_buffer { public: static mutable_byte_buffer create(); static mutable_byte_buffer create(size_t size); + static mutable_byte_buffer create_zeroed(size_t size); static mutable_byte_buffer create_reserve(size_t size); static mutable_byte_buffer create(std::string_view data); static mutable_byte_buffer create(std::span data); - static mutable_byte_buffer create(std::vector&& data); + static mutable_byte_buffer create(internal::malloc_buffer&& data); }; } // namespace dwarfs diff --git a/include/dwarfs/vector_byte_buffer_factory.h b/include/dwarfs/malloc_byte_buffer_factory.h similarity index 97% rename from include/dwarfs/vector_byte_buffer_factory.h rename to include/dwarfs/malloc_byte_buffer_factory.h index b6c90179..72e7dc71 100644 --- a/include/dwarfs/vector_byte_buffer_factory.h +++ b/include/dwarfs/malloc_byte_buffer_factory.h @@ -32,7 +32,7 @@ namespace dwarfs { -class vector_byte_buffer_factory { +class malloc_byte_buffer_factory { public: static byte_buffer_factory create(); }; diff --git a/src/block_decompressor.cpp b/src/block_decompressor.cpp index ab4a3245..1a99de63 100644 --- a/src/block_decompressor.cpp +++ b/src/block_decompressor.cpp @@ -29,7 +29,7 @@ #include #include #include -#include +#include namespace dwarfs { @@ -42,7 +42,7 @@ shared_byte_buffer block_decompressor::decompress(compression_type type, std::span data) { block_decompressor bd(type, data); - auto target = vector_byte_buffer::create_reserve(bd.uncompressed_size()); + auto target = malloc_byte_buffer::create_reserve(bd.uncompressed_size()); bd.start_decompression(target); bd.decompress_frame(bd.uncompressed_size()); return target.share(); diff --git a/src/compression/brotli.cpp b/src/compression/brotli.cpp index 08c67050..c0cbfe6f 100644 --- a/src/compression/brotli.cpp +++ b/src/compression/brotli.cpp @@ -37,9 +37,9 @@ #include #include #include +#include #include #include -#include #include "base.h" @@ -61,7 +61,7 @@ class brotli_block_compressor final : public block_compressor::impl { shared_byte_buffer compress(shared_byte_buffer const& data, std::string const* /*metadata*/) const override { - auto compressed = vector_byte_buffer::create(); // TODO: make configurable + auto compressed = malloc_byte_buffer::create(); // TODO: make configurable compressed.resize(varint::max_size + ::BrotliEncoderMaxCompressedSize(data.size())); size_t size_size = varint::encode(data.size(), compressed.data()); diff --git a/src/compression/flac.cpp b/src/compression/flac.cpp index 0481d05d..f56cf4bd 100644 --- a/src/compression/flac.cpp +++ b/src/compression/flac.cpp @@ -42,10 +42,10 @@ #include #include #include +#include #include #include #include -#include #include @@ -281,7 +281,7 @@ class flac_block_compressor final : public block_compressor::impl { pcm_pad = pcm_sample_padding::Msb; } - auto compressed = vector_byte_buffer::create(); // TODO: make configurable + auto compressed = malloc_byte_buffer::create(); // TODO: make configurable { using namespace ::apache::thrift; diff --git a/src/compression/lz4.cpp b/src/compression/lz4.cpp index 6929594d..74a6ce7d 100644 --- a/src/compression/lz4.cpp +++ b/src/compression/lz4.cpp @@ -36,8 +36,8 @@ #include #include #include +#include #include -#include #include "base.h" @@ -82,7 +82,7 @@ class lz4_block_compressor final : public block_compressor::impl { shared_byte_buffer compress(shared_byte_buffer const& data, std::string const* /*metadata*/) const override { - auto compressed = vector_byte_buffer::create(); // TODO: make configurable + auto compressed = malloc_byte_buffer::create(); // TODO: make configurable compressed.resize(sizeof(uint32_t) + LZ4_compressBound(to(data.size()))); // TODO: this should have been a varint; also, if we ever support diff --git a/src/compression/lzma.cpp b/src/compression/lzma.cpp index e3482246..d13dc673 100644 --- a/src/compression/lzma.cpp +++ b/src/compression/lzma.cpp @@ -42,10 +42,10 @@ #include #include #include +#include #include #include #include -#include #include "base.h" @@ -221,7 +221,7 @@ lzma_block_compressor::compress(shared_byte_buffer const& data, lzma_action action = LZMA_FINISH; - auto compressed = vector_byte_buffer::create(); // TODO: make configurable + auto compressed = malloc_byte_buffer::create(); // TODO: make configurable compressed.resize(data.size() - 1); s.next_in = data.data(); diff --git a/src/compression/ricepp.cpp b/src/compression/ricepp.cpp index 309eac91..2645d6f0 100644 --- a/src/compression/ricepp.cpp +++ b/src/compression/ricepp.cpp @@ -38,9 +38,9 @@ #include #include #include +#include #include #include -#include #include @@ -99,7 +99,7 @@ class ricepp_block_compressor final : public block_compressor::impl { .unused_lsb_count = static_cast(unused_lsb_count), }); - auto compressed = vector_byte_buffer::create(); // TODO: make configurable + auto compressed = malloc_byte_buffer::create(); // TODO: make configurable // TODO: see if we can resize just once... // TODO: maybe the mutable_byte_buffer interface can have .append()? diff --git a/src/compression/zstd.cpp b/src/compression/zstd.cpp index 842bafee..645cf08d 100644 --- a/src/compression/zstd.cpp +++ b/src/compression/zstd.cpp @@ -36,8 +36,8 @@ #include #include #include +#include #include -#include #include #include "base.h" @@ -101,7 +101,7 @@ class zstd_block_compressor final : public block_compressor::impl { shared_byte_buffer zstd_block_compressor::compress(shared_byte_buffer const& data, std::string const* /*metadata*/) const { - auto compressed = vector_byte_buffer::create(); // TODO: make configurable + auto compressed = malloc_byte_buffer::create(); // TODO: make configurable compressed.resize(ZSTD_compressBound(data.size())); auto ctx = ctxmgr_->make_context(); auto size = ZSTD_compressCCtx(ctx.get(), compressed.data(), compressed.size(), diff --git a/src/history.cpp b/src/history.cpp index dac511f5..13f635c0 100644 --- a/src/history.cpp +++ b/src/history.cpp @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include @@ -91,7 +91,7 @@ size_t history::size() const { return history_->entries()->size(); } shared_byte_buffer history::serialize() const { std::string buf; ::apache::thrift::CompactSerializer::serialize(*history_, &buf); - return vector_byte_buffer::create(buf).share(); + return malloc_byte_buffer::create(buf).share(); } void history::dump(std::ostream& os) const { diff --git a/src/internal/malloc_buffer.cpp b/src/internal/malloc_buffer.cpp new file mode 100644 index 00000000..5beb7c2c --- /dev/null +++ b/src/internal/malloc_buffer.cpp @@ -0,0 +1,85 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the “Software”), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * SPDX-License-Identifier: MIT + */ + +#include +#include +#include + +#include + +namespace dwarfs::internal { + +malloc_buffer::malloc_buffer(size_t size) + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc) + : data_{static_cast(std::malloc(size))} + , size_{size} + , capacity_{size} { + if (!data_) { + throw std::bad_alloc(); + } +} + +malloc_buffer::malloc_buffer(void const* data, size_t size) + : malloc_buffer(size) { + copy(data_, data, size); +} + +malloc_buffer::malloc_buffer(std::span data) + : malloc_buffer(data.size_bytes()) { + copy(data_, data.data(), data.size_bytes()); +} + +malloc_buffer::~malloc_buffer() { + if (data_) { + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory) + std::free(data_); + } +} + +void malloc_buffer::resize_buffer(size_t new_size) { + // NOLINTNEXTLINE(cppcoreguidelines-no-malloc,cppcoreguidelines-owning-memory) + auto new_data = static_cast(std::realloc(data_, new_size)); + if (!new_data) { + throw std::bad_alloc(); + } + data_ = new_data; + capacity_ = new_size; +} + +void malloc_buffer::grow(size_t new_size) { + assert(new_size > size_); + resize_buffer(new_size); +} + +void malloc_buffer::shrink_to_fit() { + if (size_ < capacity_) { + resize_buffer(size_); + } +} + +} // namespace dwarfs::internal diff --git a/src/vector_byte_buffer.cpp b/src/malloc_byte_buffer.cpp similarity index 66% rename from src/vector_byte_buffer.cpp rename to src/malloc_byte_buffer.cpp index 02fe9be1..76eb3862 100644 --- a/src/vector_byte_buffer.cpp +++ b/src/malloc_byte_buffer.cpp @@ -27,27 +27,29 @@ */ #include +#include #include #include #include -#include -#include +#include + +#include namespace dwarfs { namespace { -class vector_byte_buffer_impl : public mutable_byte_buffer_interface { +class malloc_byte_buffer_impl : public mutable_byte_buffer_interface { public: - vector_byte_buffer_impl() = default; - explicit vector_byte_buffer_impl(size_t size) + malloc_byte_buffer_impl() = default; + explicit malloc_byte_buffer_impl(size_t size) : data_(size) {} - explicit vector_byte_buffer_impl(std::string_view data) - : data_{data.begin(), data.end()} {} - explicit vector_byte_buffer_impl(std::span data) - : data_{data.begin(), data.end()} {} - explicit vector_byte_buffer_impl(std::vector&& data) + explicit malloc_byte_buffer_impl(std::string_view data) + : data_{data.data(), data.size()} {} + explicit malloc_byte_buffer_impl(std::span data) + : data_{data} {} + explicit malloc_byte_buffer_impl(internal::malloc_buffer&& data) : data_{std::move(data)} {} size_t size() const override { return data_.size(); } @@ -92,7 +94,7 @@ class vector_byte_buffer_impl : public mutable_byte_buffer_interface { frozen_.store(true, std::memory_order_release); } - std::vector& raw_vector() override { return data_; } + internal::malloc_buffer& raw_buffer() override { return data_; } private: void assert_not_frozen(std::string_view what) const { @@ -108,38 +110,44 @@ class vector_byte_buffer_impl : public mutable_byte_buffer_interface { bool frozen() const { return frozen_.load(std::memory_order_acquire); } - std::vector data_; + internal::malloc_buffer data_; std::atomic frozen_{false}; static_assert(std::atomic::is_always_lock_free); }; } // namespace -mutable_byte_buffer vector_byte_buffer::create() { - return mutable_byte_buffer{std::make_shared()}; +mutable_byte_buffer malloc_byte_buffer::create() { + return mutable_byte_buffer{std::make_shared()}; } -mutable_byte_buffer vector_byte_buffer::create(size_t size) { - return mutable_byte_buffer{std::make_shared(size)}; +mutable_byte_buffer malloc_byte_buffer::create(size_t size) { + return mutable_byte_buffer{std::make_shared(size)}; } -mutable_byte_buffer vector_byte_buffer::create_reserve(size_t size) { - auto rv = std::make_shared(); +mutable_byte_buffer malloc_byte_buffer::create_zeroed(size_t size) { + auto buffer = std::make_shared(size); + std::memset(buffer->mutable_data(), 0, size); + return mutable_byte_buffer{std::move(buffer)}; +} + +mutable_byte_buffer malloc_byte_buffer::create_reserve(size_t size) { + auto rv = std::make_shared(); rv->reserve(size); return mutable_byte_buffer{std::move(rv)}; } -mutable_byte_buffer vector_byte_buffer::create(std::string_view data) { - return mutable_byte_buffer{std::make_shared(data)}; +mutable_byte_buffer malloc_byte_buffer::create(std::string_view data) { + return mutable_byte_buffer{std::make_shared(data)}; } -mutable_byte_buffer vector_byte_buffer::create(std::span data) { - return mutable_byte_buffer{std::make_shared(data)}; +mutable_byte_buffer malloc_byte_buffer::create(std::span data) { + return mutable_byte_buffer{std::make_shared(data)}; } -mutable_byte_buffer vector_byte_buffer::create(std::vector&& data) { +mutable_byte_buffer malloc_byte_buffer::create(internal::malloc_buffer&& data) { return mutable_byte_buffer{ - std::make_shared(std::move(data))}; + std::make_shared(std::move(data))}; } } // namespace dwarfs diff --git a/src/vector_byte_buffer_factory.cpp b/src/malloc_byte_buffer_factory.cpp similarity index 82% rename from src/vector_byte_buffer_factory.cpp rename to src/malloc_byte_buffer_factory.cpp index f462c68e..8a8c6353 100644 --- a/src/vector_byte_buffer_factory.cpp +++ b/src/malloc_byte_buffer_factory.cpp @@ -26,25 +26,25 @@ * SPDX-License-Identifier: MIT */ -#include -#include +#include +#include namespace dwarfs { namespace { -class vector_byte_buffer_factory_impl : public byte_buffer_factory_interface { +class malloc_byte_buffer_factory_impl : public byte_buffer_factory_interface { public: mutable_byte_buffer create_mutable_fixed_reserve(size_t size) const override { - return vector_byte_buffer::create_reserve(size); + return malloc_byte_buffer::create_reserve(size); } }; } // namespace -byte_buffer_factory vector_byte_buffer_factory::create() { +byte_buffer_factory malloc_byte_buffer_factory::create() { return byte_buffer_factory{ - std::make_shared()}; + std::make_shared()}; } } // namespace dwarfs diff --git a/src/reader/block_cache_byte_buffer_factory.cpp b/src/reader/block_cache_byte_buffer_factory.cpp index e27d2012..53f5d7a0 100644 --- a/src/reader/block_cache_byte_buffer_factory.cpp +++ b/src/reader/block_cache_byte_buffer_factory.cpp @@ -36,8 +36,8 @@ #include #endif +#include #include -#include namespace dwarfs::reader { @@ -138,9 +138,9 @@ class mmap_byte_buffer_impl : public mutable_byte_buffer_interface { // always frozen } - std::vector& raw_vector() override { + internal::malloc_buffer& raw_buffer() override { throw std::runtime_error( - "operation not allowed on mmap buffer: raw_vector"); + "operation not allowed on mmap buffer: raw_buffer"); } private: @@ -163,7 +163,7 @@ class block_cache_byte_buffer_factory_impl if (mode_ == block_cache_allocation_mode::MMAP) { return mutable_byte_buffer{std::make_shared(size)}; } - return vector_byte_buffer::create_reserve(size); + return malloc_byte_buffer::create_reserve(size); } private: diff --git a/src/reader/filesystem_v2.cpp b/src/reader/filesystem_v2.cpp index 6d8787e2..baf416a1 100644 --- a/src/reader/filesystem_v2.cpp +++ b/src/reader/filesystem_v2.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -53,7 +54,6 @@ #include #include #include -#include #include #include diff --git a/src/writer/filesystem_writer.cpp b/src/writer/filesystem_writer.cpp index 4a53b977..217c7c41 100644 --- a/src/writer/filesystem_writer.cpp +++ b/src/writer/filesystem_writer.cpp @@ -40,9 +40,9 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -395,7 +395,7 @@ class rewritten_fsblock : public fsblock::impl { { // TODO: we don't have to do this for uncompressed blocks block_decompressor bd(data_comp_type_, data_); - auto buffer = bd.start_decompression(vector_byte_buffer::create()); + auto buffer = bd.start_decompression(malloc_byte_buffer::create()); bd.decompress_frame(bd.uncompressed_size()); if (!meta) { diff --git a/src/writer/internal/metadata_freezer.cpp b/src/writer/internal/metadata_freezer.cpp index 6df741e6..c761672d 100644 --- a/src/writer/internal/metadata_freezer.cpp +++ b/src/writer/internal/metadata_freezer.cpp @@ -24,7 +24,7 @@ #include #include -#include +#include #include @@ -47,9 +47,9 @@ std::pair freeze_to_buffer(T const& x) { std::string schema; serializeRootLayout(layout, schema); - auto schema_buffer = vector_byte_buffer::create(schema); + auto schema_buffer = malloc_byte_buffer::create(schema); - auto data_buffer = vector_byte_buffer::create(content_size); + auto data_buffer = malloc_byte_buffer::create_zeroed(content_size); folly::MutableByteRange content_range(data_buffer.data(), data_buffer.size()); ByteRangeFreezer::freeze(layout, x, content_range); diff --git a/src/writer/segmenter.cpp b/src/writer/segmenter.cpp index befbc8e1..ad33dbac 100644 --- a/src/writer/segmenter.cpp +++ b/src/writer/segmenter.cpp @@ -47,11 +47,12 @@ #include #include #include +#include #include -#include #include #include +#include #include #include #include @@ -559,13 +560,10 @@ class basic_granular_container_adapter : private GranularityPolicy { T& v_; }; -template -using granular_vector_adapter = - basic_granular_container_adapter, GranularityPolicy>; - template -using granular_byte_buffer_adapter = - basic_granular_container_adapter; +using granular_buffer_adapter = + basic_granular_container_adapter; template class active_block : private GranularityPolicy { @@ -590,7 +588,7 @@ class active_block : private GranularityPolicy { , filter_(bloom_filter_size) , repseqmap_{repseqmap} , repeating_collisions_{repcoll} - , data_{vector_byte_buffer::create()} { + , data_{malloc_byte_buffer::create()} { DWARFS_CHECK((window_step & window_step_mask_) == 0, "window step size not a power of two"); data_.reserve(this->frames_to_bytes(capacity_in_frames_)); @@ -894,8 +892,8 @@ active_block::append_bytes( auto src = this->template create< granular_span_adapter>(data); - auto v = this->template create< - granular_vector_adapter>(data_.raw_vector()); + auto v = this->template create>( + data_.raw_buffer()); // TODO: this works in theory, but slows down the segmenter by almost 10% // auto v = this->template create< @@ -936,9 +934,8 @@ template void segment_match::verify_and_extend( granular_span_adapter const& data, size_t pos, size_t len, size_t begin, size_t end) { - auto v = this->template create< - granular_vector_adapter>( - block_->data().raw_vector()); + auto v = this->template create>( + block_->data().raw_buffer()); // First, check if the regions actually match if (v.compare(offset_, data.subspan(pos, len)) == 0) { diff --git a/test/flac_compressor_test.cpp b/test/flac_compressor_test.cpp index 32e5bfff..3eec4c25 100644 --- a/test/flac_compressor_test.cpp +++ b/test/flac_compressor_test.cpp @@ -30,8 +30,8 @@ #include #include +#include #include -#include using namespace dwarfs; @@ -74,7 +74,7 @@ make_test_data(int channels, int samples, int bytes, int bits, make_sine(bits, samples, 3.1 * ((599 * (c + 1)) % 256))); } auto muxed = multiplex(data); - auto out = vector_byte_buffer::create(); + auto out = malloc_byte_buffer::create(); out.resize(bytes * channels * samples); pcm_sample_transformer xfm(end, sig, pad, bytes, bits); xfm.pack(out.span(), muxed); diff --git a/test/ricepp_compressor_test.cpp b/test/ricepp_compressor_test.cpp index 761c7c0e..0c8f7841 100644 --- a/test/ricepp_compressor_test.cpp +++ b/test/ricepp_compressor_test.cpp @@ -40,7 +40,7 @@ #include #include -#include +#include using namespace dwarfs; @@ -96,7 +96,7 @@ shared_byte_buffer make_test_data(int components, int pixels, int unused_lsb) { } } - auto out = vector_byte_buffer::create(); + auto out = malloc_byte_buffer::create(); out.resize(tmp.size() * sizeof(ValueType)); std::memcpy(out.data(), tmp.data(), out.size());