mirror of
https://github.com/mhx/dwarfs.git
synced 2025-08-04 02:06:22 -04:00
feat: add block_cache_allocation_mode
and Windows mmap wrapper
This commit is contained in:
parent
0e18c26e05
commit
d739973d00
@ -30,11 +30,14 @@
|
|||||||
|
|
||||||
#include <dwarfs/byte_buffer_factory.h>
|
#include <dwarfs/byte_buffer_factory.h>
|
||||||
|
|
||||||
|
#include <dwarfs/reader/block_cache_options.h>
|
||||||
|
|
||||||
namespace dwarfs::reader {
|
namespace dwarfs::reader {
|
||||||
|
|
||||||
class block_cache_byte_buffer_factory {
|
class block_cache_byte_buffer_factory {
|
||||||
public:
|
public:
|
||||||
static byte_buffer_factory create();
|
static byte_buffer_factory create();
|
||||||
|
static byte_buffer_factory create(block_cache_allocation_mode mode);
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace dwarfs::reader
|
} // namespace dwarfs::reader
|
||||||
|
@ -33,6 +33,11 @@
|
|||||||
|
|
||||||
namespace dwarfs::reader {
|
namespace dwarfs::reader {
|
||||||
|
|
||||||
|
enum class block_cache_allocation_mode {
|
||||||
|
MALLOC,
|
||||||
|
MMAP,
|
||||||
|
};
|
||||||
|
|
||||||
struct block_cache_options {
|
struct block_cache_options {
|
||||||
size_t max_bytes{static_cast<size_t>(512) << 20};
|
size_t max_bytes{static_cast<size_t>(512) << 20};
|
||||||
size_t num_workers{0};
|
size_t num_workers{0};
|
||||||
@ -41,6 +46,8 @@ struct block_cache_options {
|
|||||||
bool init_workers{true};
|
bool init_workers{true};
|
||||||
bool disable_block_integrity_check{false};
|
bool disable_block_integrity_check{false};
|
||||||
size_t sequential_access_detector_threshold{0};
|
size_t sequential_access_detector_threshold{0};
|
||||||
|
block_cache_allocation_mode allocation_mode{
|
||||||
|
block_cache_allocation_mode::MALLOC};
|
||||||
};
|
};
|
||||||
|
|
||||||
std::ostream& operator<<(std::ostream& os, block_cache_options const& opts);
|
std::ostream& operator<<(std::ostream& os, block_cache_options const& opts);
|
||||||
|
@ -28,23 +28,52 @@
|
|||||||
|
|
||||||
#include <cassert>
|
#include <cassert>
|
||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
|
#include <system_error>
|
||||||
|
|
||||||
#ifdef _WIN32
|
#ifdef _WIN32
|
||||||
#include <dwarfs/vector_byte_buffer.h>
|
#include <folly/portability/Windows.h>
|
||||||
#else
|
#else
|
||||||
#include <sys/mman.h>
|
#include <sys/mman.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#include <dwarfs/reader/block_cache_byte_buffer_factory.h>
|
#include <dwarfs/reader/block_cache_byte_buffer_factory.h>
|
||||||
|
#include <dwarfs/vector_byte_buffer.h>
|
||||||
|
|
||||||
namespace dwarfs::reader {
|
namespace dwarfs::reader {
|
||||||
|
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
#ifndef _WIN32
|
#ifdef _WIN32
|
||||||
class mmap_file {
|
class mmap_block {
|
||||||
public:
|
public:
|
||||||
mmap_file(size_t size)
|
mmap_block(size_t size)
|
||||||
|
: data_{::VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT,
|
||||||
|
PAGE_READWRITE)}
|
||||||
|
, size_{size} {
|
||||||
|
if (!data_) {
|
||||||
|
std::error_code ec(::GetLastError(), std::system_category());
|
||||||
|
throw std::runtime_error("VirtualAlloc failed: " + ec.message());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t size() const { return size_; }
|
||||||
|
|
||||||
|
uint8_t* data() { return static_cast<uint8_t*>(data_); }
|
||||||
|
uint8_t const* data() const { return static_cast<uint8_t const*>(data_); }
|
||||||
|
|
||||||
|
~mmap_block() {
|
||||||
|
auto rv [[maybe_unused]] = ::VirtualFree(data_, 0, MEM_RELEASE);
|
||||||
|
assert(rv);
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
void* data_;
|
||||||
|
size_t size_;
|
||||||
|
};
|
||||||
|
#else
|
||||||
|
class mmap_block {
|
||||||
|
public:
|
||||||
|
mmap_block(size_t size)
|
||||||
: data_{::mmap(nullptr, size, PROT_READ | PROT_WRITE,
|
: data_{::mmap(nullptr, size, PROT_READ | PROT_WRITE,
|
||||||
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)}
|
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)}
|
||||||
, size_{size} {
|
, size_{size} {
|
||||||
@ -58,7 +87,7 @@ class mmap_file {
|
|||||||
uint8_t* data() { return static_cast<uint8_t*>(data_); }
|
uint8_t* data() { return static_cast<uint8_t*>(data_); }
|
||||||
uint8_t const* data() const { return static_cast<uint8_t const*>(data_); }
|
uint8_t const* data() const { return static_cast<uint8_t const*>(data_); }
|
||||||
|
|
||||||
~mmap_file() {
|
~mmap_block() {
|
||||||
auto rv [[maybe_unused]] = ::munmap(data_, size_);
|
auto rv [[maybe_unused]] = ::munmap(data_, size_);
|
||||||
assert(rv == 0);
|
assert(rv == 0);
|
||||||
}
|
}
|
||||||
@ -67,6 +96,7 @@ class mmap_file {
|
|||||||
void* data_;
|
void* data_;
|
||||||
size_t size_;
|
size_t size_;
|
||||||
};
|
};
|
||||||
|
#endif
|
||||||
|
|
||||||
class mmap_byte_buffer_impl : public mutable_byte_buffer_interface {
|
class mmap_byte_buffer_impl : public mutable_byte_buffer_interface {
|
||||||
public:
|
public:
|
||||||
@ -119,28 +149,39 @@ class mmap_byte_buffer_impl : public mutable_byte_buffer_interface {
|
|||||||
std::string{what});
|
std::string{what});
|
||||||
}
|
}
|
||||||
|
|
||||||
mmap_file data_;
|
mmap_block data_;
|
||||||
size_t size_{0};
|
size_t size_{0};
|
||||||
};
|
};
|
||||||
#endif
|
|
||||||
|
|
||||||
class block_cache_byte_buffer_factory_impl
|
class block_cache_byte_buffer_factory_impl
|
||||||
: public byte_buffer_factory_interface {
|
: public byte_buffer_factory_interface {
|
||||||
public:
|
public:
|
||||||
|
block_cache_byte_buffer_factory_impl(block_cache_allocation_mode mode)
|
||||||
|
: mode_{mode} {}
|
||||||
|
|
||||||
mutable_byte_buffer create_mutable_fixed_reserve(size_t size) const override {
|
mutable_byte_buffer create_mutable_fixed_reserve(size_t size) const override {
|
||||||
#ifdef _WIN32
|
if (mode_ == block_cache_allocation_mode::MMAP) {
|
||||||
return vector_byte_buffer::create_reserve(size);
|
|
||||||
#else
|
|
||||||
return mutable_byte_buffer{std::make_shared<mmap_byte_buffer_impl>(size)};
|
return mutable_byte_buffer{std::make_shared<mmap_byte_buffer_impl>(size)};
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
|
return vector_byte_buffer::create_reserve(size);
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
block_cache_allocation_mode mode_;
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
byte_buffer_factory block_cache_byte_buffer_factory::create() {
|
byte_buffer_factory block_cache_byte_buffer_factory::create() {
|
||||||
return byte_buffer_factory{
|
return byte_buffer_factory{
|
||||||
std::make_shared<block_cache_byte_buffer_factory_impl>()};
|
std::make_shared<block_cache_byte_buffer_factory_impl>(
|
||||||
|
block_cache_allocation_mode::MALLOC)};
|
||||||
|
}
|
||||||
|
|
||||||
|
byte_buffer_factory
|
||||||
|
block_cache_byte_buffer_factory::create(block_cache_allocation_mode mode) {
|
||||||
|
return byte_buffer_factory{
|
||||||
|
std::make_shared<block_cache_byte_buffer_factory_impl>(mode)};
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace dwarfs::reader
|
} // namespace dwarfs::reader
|
||||||
|
Loading…
x
Reference in New Issue
Block a user