TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-09-14 06:48:39 -04:00
Code Issues Packages Projects Releases Wiki Activity

File system image structure, version 2, part 1

Browse Source
This commit is contained in:
Marcus Holland-Moritz 2020-12-16 22:58:53 +01:00
parent 2f3f1fa946
commit cde36cffc4
2 changed files with 45 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
include/dwarfs/fstypes.h
View File

@ -32,6 +32,7 @@
#include <folly/small_vector.h> #include <folly/small_vector.h>
#include "dwarfs/block_compressor.h" // TODO: or the other way round? #include "dwarfs/block_compressor.h" // TODO: or the other way round?
#include "dwarfs/checksum.h"
namespace dwarfs { namespace dwarfs {
@ -64,7 +65,7 @@ struct iovec_read_buf {
}; };
constexpr uint8_t MAJOR_VERSION = 2; constexpr uint8_t MAJOR_VERSION = 2;
constexpr uint8_t MINOR_VERSION = 1; constexpr uint8_t MINOR_VERSION = 2;
enum class section_type : uint16_t { enum class section_type : uint16_t {
BLOCK = 0, BLOCK = 0,
@ -93,6 +94,26 @@ struct section_header {
void dump(std::ostream& os) const; void dump(std::ostream& os) const;
}; };
struct section_header_v2 {
char magic[6]; // [0] "DWARFS" / file_header no longer needed
uint8_t major; // [6] major version
uint8_t minor; // [7] minor version
uint8_t sha2_512_256[32]; // [8] SHA2-512/256 starting from next field
uint64_t xxh3_64; // [40] XXH3-64 starting from next field
uint32_t number; // [48] section number
uint16_t type; // [52] section type
uint16_t compression; // [54] compression
uint64_t length; // [56] length of section
static_assert(checksum::digest_size(checksum::algorithm::XXH3_64) ==
sizeof(xxh3_64));
static_assert(checksum::digest_size(checksum::algorithm::SHA2_512_256) ==
sizeof(sha2_512_256));
std::string to_string() const;
void dump(std::ostream& os) const;
};
std::string get_compression_name(compression_type type); std::string get_compression_name(compression_type type);
std::string get_section_name(section_type type); std::string get_section_name(section_type type);

40
src/dwarfs/filesystem_writer.cpp
View File

@ -31,6 +31,7 @@
#include <folly/system/ThreadName.h> #include <folly/system/ThreadName.h>
#include "dwarfs/block_compressor.h" #include "dwarfs/block_compressor.h"
#include "dwarfs/checksum.h"
#include "dwarfs/filesystem_writer.h" #include "dwarfs/filesystem_writer.h"
#include "dwarfs/fstypes.h" #include "dwarfs/fstypes.h"
#include "dwarfs/logger.h" #include "dwarfs/logger.h"
@ -155,7 +156,6 @@ class filesystem_writer_ : public filesystem_writer::impl {
template <typename T> template <typename T>
void write(const T& obj); void write(const T& obj);
void write(const std::vector<uint8_t>& data); void write(const std::vector<uint8_t>& data);
void write_file_header();
void writer_thread(); void writer_thread();
size_t mem_used() const; size_t mem_used() const;
@ -172,6 +172,7 @@ class filesystem_writer_ : public filesystem_writer::impl {
std::condition_variable cond_; std::condition_variable cond_;
volatile bool flush_; volatile bool flush_;
std::thread writer_thread_; std::thread writer_thread_;
uint32_t section_number_{0};
}; };
template <typename LoggerPolicy> template <typename LoggerPolicy>
@ -188,9 +189,7 @@ filesystem_writer_<LoggerPolicy>::filesystem_writer_(
, max_queue_size_(max_queue_size) , max_queue_size_(max_queue_size)
, log_(lgr) , log_(lgr)
, flush_(false) , flush_(false)
, writer_thread_(&filesystem_writer_::writer_thread, this) { , writer_thread_(&filesystem_writer_::writer_thread, this) {}
write_file_header();
}
template <typename LoggerPolicy> template <typename LoggerPolicy>
filesystem_writer_<LoggerPolicy>::~filesystem_writer_() noexcept { filesystem_writer_<LoggerPolicy>::~filesystem_writer_() noexcept {
@ -268,24 +267,31 @@ void filesystem_writer_<LoggerPolicy>::write(const std::vector<uint8_t>& data) {
write(reinterpret_cast<const char*>(&data[0]), data.size()); write(reinterpret_cast<const char*>(&data[0]), data.size());
} }
template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write_file_header() {
file_header hdr;
::memcpy(&hdr.magic[0], "DWARFS", 6);
hdr.major = MAJOR_VERSION;
hdr.minor = MINOR_VERSION;
write(hdr);
}
template <typename LoggerPolicy> template <typename LoggerPolicy>
void filesystem_writer_<LoggerPolicy>::write(section_type type, void filesystem_writer_<LoggerPolicy>::write(section_type type,
compression_type compression, compression_type compression,
const std::vector<uint8_t>& data) { const std::vector<uint8_t>& data) {
section_header sh; section_header_v2 sh;
sh.type = type; ::memcpy(&sh.magic[0], "DWARFS", 6);
sh.compression = compression; sh.major = MAJOR_VERSION;
sh.unused = 0; sh.minor = MINOR_VERSION;
sh.number = section_number_++;
sh.type = static_cast<uint16_t>(type);
sh.compression = static_cast<uint16_t>(compression);
sh.length = data.size(); sh.length = data.size();
checksum xxh(checksum::algorithm::XXH3_64);
xxh.update(&sh.number,
sizeof(section_header_v2) - offsetof(section_header_v2, number));
xxh.update(data.data(), data.size());
DWARFS_CHECK(xxh.finalize(&sh.xxh3_64), "XXH3-64 checksum failed");
checksum sha(checksum::algorithm::SHA2_512_256);
sha.update(&sh.xxh3_64,
sizeof(section_header_v2) - offsetof(section_header_v2, xxh3_64));
sha.update(data.data(), data.size());
DWARFS_CHECK(sha.finalize(&sh.sha2_512_256), "SHA512/256 checksum failed");
write(sh); write(sh);
write(data); write(data);