TheCloud/dwarfs
1
0
Fork 0
mirror of https://github.com/mhx/dwarfs.git synced 2025-08-03 09:47:01 -04:00
Code Issues Packages Projects Releases Wiki Activity
dwarfs/test/compat_test.cpp
Marcus Holland-Moritz e6ae08475a feat: allow disabling advanced features in dwarfs-fuse-extract
2025-04-13 22:14:57 +02:00

1442 lines
50 KiB
C++
Raw Permalink Blame History

/* vim:set ts=2 sw=2 sts=2 et: */
/**
* \author Marcus Holland-Moritz (github@mhxnet.de)
* \copyright Copyright (c) Marcus Holland-Moritz
*
* This file is part of dwarfs.
*
* dwarfs is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* dwarfs is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with dwarfs. If not, see <https://www.gnu.org/licenses/>.
*
* SPDX-License-Identifier: GPL-3.0-only
*/
#include <gtest/gtest.h>
// TODO: this test should be autogenerated somehow...
#include <algorithm>
#include <array>
#include <cstring>
#include <filesystem>
#include <map>
#include <sstream>
#include <string>
#include <tuple>
#include <vector>
#include <folly/FileUtil.h>
#include <folly/String.h>
#include <dwarfs/block_compressor.h>
#include <dwarfs/config.h>
#include <dwarfs/file_stat.h>
#include <dwarfs/logger.h>
#include <dwarfs/mmap.h>
#include <dwarfs/reader/filesystem_options.h>
#include <dwarfs/reader/filesystem_v2.h>
#include <dwarfs/reader/fsinfo_options.h>
#include <dwarfs/string.h>
#include <dwarfs/thread_pool.h>
#include <dwarfs/utility/filesystem_extractor.h>
#include <dwarfs/utility/rewrite_filesystem.h>
#include <dwarfs/utility/rewrite_options.h>
#include <dwarfs/vfs_stat.h>
#include <dwarfs/writer/filesystem_block_category_resolver.h>
#include <dwarfs/writer/filesystem_writer.h>
#include <dwarfs/writer/filesystem_writer_options.h>
#include <dwarfs/writer/writer_progress.h>
#include "mmap_mock.h"
#include "test_helpers.h"
#include "test_logger.h"
using namespace dwarfs;
namespace fs = std::filesystem;
/*------------------------------------------------------------------------------
Command line options used to create the 0.4.1 and later images:
$ mkdwarfs -i @compat -o compat-vx.y.z.dwarfs --with-devices --with-specials \
-S10 -l0 -W5 --order=similarity
------------------------------------------------------------------------------*/
namespace {
auto test_dir = fs::path(TEST_DATA_DIR).make_preferred();
template <class F>
void walk_json(nlohmann::json& j, F f) {
for (auto& [k, v] : j.items()) {
f(v);
if (v.is_structured()) {
walk_json(v, f);
}
}
}
void remove_inode_numbers(nlohmann::json& j) {
walk_json(j, [](nlohmann::json& j) {
if (j.contains("inode")) {
j.erase("inode");
}
});
}
char const* reference_v0_2 = R"(
{
"root": {
"inode": 0,
"inodes": [
{
"inode": 32,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "bench.sh",
"size": 1517,
"type": "file"
},
{
"inode": 1,
"inodes": [],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "dev",
"type": "directory"
},
{
"inode": 2,
"inodes": [
{
"inode": 3,
"inodes": [],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "alsoempty",
"type": "directory"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "empty",
"type": "directory"
},
{
"inode": 4,
"inodes": [
{
"inode": 5,
"inodes": [
{
"inode": 6,
"inodes": [
{
"inode": 7,
"inodes": [
{
"inode": 8,
"inodes": [
{
"inode": 9,
"inodes": [
{
"inode": 10,
"inodes": [
{
"inode": 11,
"inodes": [
{
"inode": 12,
"inodes": [
{
"inode": 13,
"inodes": [
{
"inode": 17,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "a",
"size": 2,
"type": "file"
},
{
"inode": 18,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "b",
"size": 2,
"type": "file"
},
{
"inode": 32,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "blubb",
"size": 1517,
"type": "file"
},
{
"inode": 19,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "c",
"size": 2,
"type": "file"
},
{
"inode": 20,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "d",
"size": 2,
"type": "file"
},
{
"inode": 21,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "e",
"size": 2,
"type": "file"
},
{
"inode": 22,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "f",
"size": 2,
"type": "file"
},
{
"inode": 23,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "g",
"size": 2,
"type": "file"
},
{
"inode": 24,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "h",
"size": 2,
"type": "file"
},
{
"inode": 25,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "i",
"size": 2,
"type": "file"
},
{
"inode": 26,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "j",
"size": 2,
"type": "file"
},
{
"inode": 27,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "k",
"size": 2,
"type": "file"
},
{
"inode": 28,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "l",
"size": 2,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "9",
"type": "directory"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "8",
"type": "directory"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "7",
"type": "directory"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "6",
"type": "directory"
},
{
"inode": 32,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "z",
"size": 1517,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "5",
"type": "directory"
},
{
"inode": 32,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "y",
"size": 1517,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "4",
"type": "directory"
},
{
"inode": 29,
"mode": 33261,
"modestring": "----rwxr-xr-x",
"name": "copy.sh",
"size": 94,
"type": "file"
},
{
"inode": 32,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "x",
"size": 1517,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "3",
"type": "directory"
},
{
"inode": 29,
"mode": 33261,
"modestring": "----rwxr-xr-x",
"name": "xxx.sh",
"size": 94,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "2",
"type": "directory"
},
{
"inode": 29,
"mode": 33261,
"modestring": "----rwxr-xr-x",
"name": "fmt.sh",
"size": 94,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "1",
"type": "directory"
},
{
"inode": 14,
"mode": 41471,
"modestring": "---lrwxrwxrwx",
"name": "bad",
"target": "../foo",
"type": "link"
},
{
"inode": 16,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "bar",
"size": 0,
"type": "file"
},
{
"inode": 32,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "bla.sh",
"size": 1517,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "foo",
"type": "directory"
},
{
"inode": 15,
"mode": 41471,
"modestring": "---lrwxrwxrwx",
"name": "foobar",
"target": "foo/bar",
"type": "link"
},
{
"inode": 29,
"mode": 33261,
"modestring": "----rwxr-xr-x",
"name": "format.sh",
"size": 94,
"type": "file"
},
{
"inode": 31,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "perl-exec.sh",
"size": 87,
"type": "file"
},
{
"inode": 30,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "test.py",
"size": 1012,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"type": "directory"
},
"statvfs": {
"f_blocks": 10614,
"f_bsize": 1,
"f_files": 33
}
}
)";
char const* reference_v0_4 = R"(
{
"root": {
"inode": 0,
"inodes": [
{
"inode": 32,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "bench.sh",
"size": 1517,
"type": "file"
},
{
"inode": 1,
"inodes": [
{
"device_id": 259,
"inode": 33,
"mode": 8630,
"modestring": "---crw-rw-rw-",
"name": "null",
"type": "chardev"
},
{
"device_id": 261,
"inode": 34,
"mode": 8630,
"modestring": "---crw-rw-rw-",
"name": "zero",
"type": "chardev"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "dev",
"type": "directory"
},
{
"inode": 2,
"inodes": [
{
"inode": 3,
"inodes": [],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "alsoempty",
"type": "directory"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "empty",
"type": "directory"
},
{
"inode": 4,
"inodes": [
{
"inode": 5,
"inodes": [
{
"inode": 6,
"inodes": [
{
"inode": 7,
"inodes": [
{
"inode": 8,
"inodes": [
{
"inode": 9,
"inodes": [
{
"inode": 10,
"inodes": [
{
"inode": 11,
"inodes": [
{
"inode": 12,
"inodes": [
{
"inode": 13,
"inodes": [
{
"inode": 17,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "a",
"size": 2,
"type": "file"
},
{
"inode": 18,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "b",
"size": 2,
"type": "file"
},
{
"inode": 32,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "blubb",
"size": 1517,
"type": "file"
},
{
"inode": 19,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "c",
"size": 2,
"type": "file"
},
{
"inode": 20,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "d",
"size": 2,
"type": "file"
},
{
"inode": 21,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "e",
"size": 2,
"type": "file"
},
{
"inode": 22,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "f",
"size": 2,
"type": "file"
},
{
"inode": 23,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "g",
"size": 2,
"type": "file"
},
{
"inode": 24,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "h",
"size": 2,
"type": "file"
},
{
"inode": 25,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "i",
"size": 2,
"type": "file"
},
{
"inode": 26,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "j",
"size": 2,
"type": "file"
},
{
"inode": 27,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "k",
"size": 2,
"type": "file"
},
{
"inode": 28,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "l",
"size": 2,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "9",
"type": "directory"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "8",
"type": "directory"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "7",
"type": "directory"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "6",
"type": "directory"
},
{
"inode": 32,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "z",
"size": 1517,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "5",
"type": "directory"
},
{
"inode": 32,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "y",
"size": 1517,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "4",
"type": "directory"
},
{
"inode": 29,
"mode": 33261,
"modestring": "----rwxr-xr-x",
"name": "copy.sh",
"size": 94,
"type": "file"
},
{
"inode": 32,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "x",
"size": 1517,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "3",
"type": "directory"
},
{
"inode": 29,
"mode": 33261,
"modestring": "----rwxr-xr-x",
"name": "xxx.sh",
"size": 94,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "2",
"type": "directory"
},
{
"inode": 29,
"mode": 33261,
"modestring": "----rwxr-xr-x",
"name": "fmt.sh",
"size": 94,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "1",
"type": "directory"
},
{
"inode": 14,
"mode": 41471,
"modestring": "---lrwxrwxrwx",
"name": "bad",
"target": "../foo",
"type": "link"
},
{
"inode": 16,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "bar",
"size": 0,
"type": "file"
},
{
"inode": 32,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "bla.sh",
"size": 1517,
"type": "file"
},
{
"inode": 35,
"mode": 4516,
"modestring": "---prw-r--r--",
"name": "pipe",
"type": "fifo"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"name": "foo",
"type": "directory"
},
{
"inode": 15,
"mode": 41471,
"modestring": "---lrwxrwxrwx",
"name": "foobar",
"target": "foo/bar",
"type": "link"
},
{
"inode": 29,
"mode": 33261,
"modestring": "----rwxr-xr-x",
"name": "format.sh",
"size": 94,
"type": "file"
},
{
"inode": 31,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "perl-exec.sh",
"size": 87,
"type": "file"
},
{
"inode": 30,
"mode": 33188,
"modestring": "----rw-r--r--",
"name": "test.py",
"size": 1012,
"type": "file"
}
],
"mode": 16877,
"modestring": "---drwxr-xr-x",
"type": "directory"
},
"statvfs": {
"f_blocks": 10614,
"f_bsize": 1,
"f_files": 36
}
}
)";
std::vector<std::string> versions{
"0.2.0", "0.2.3", "0.3.0", "0.4.0", "0.4.1",
"0.5.6", "0.6.2", "0.7.5", "0.8.0", "0.9.10",
};
std::string format_sh = R"(#!/bin/bash
find test/ src/ include/ -type f -name '*.[ch]*' | xargs -d $'\n' clang-format -i
)";
std::vector<std::string> headers{
"D",
"DWARFS",
format_sh,
"DWARFS" + format_sh,
"DWARFS" + format_sh + "DWARDWAR",
};
std::vector<std::string> headers_v2{
"DWARFS\x02",
"DWARFS\x02" + format_sh,
"DWARFS\x02" + format_sh + "DWARFS\x02",
};
file_stat make_stat(posix_file_type::value type, file_stat::perms_type perms,
file_stat::off_type size) {
file_stat st;
st.set_mode(type | perms);
st.set_size(size);
return st;
}
template <typename T>
void walk_tree(reader::filesystem_v2 const& fs, T& cb,
std::optional<reader::dir_entry_view> dev = std::nullopt) {
if (!dev) {
dev.emplace(fs.root());
}
cb(*dev);
auto iv = dev->inode();
if (iv.is_directory()) {
auto dir = fs.opendir(iv);
assert(dir);
for (auto e : *dir) {
walk_tree(fs, cb, e);
}
}
}
void check_compat(logger& lgr [[maybe_unused]], reader::filesystem_v2 const& fs,
std::string const& version, bool enable_nlink) {
bool const has_devices = not(version == "0.2.0" or version == "0.2.3");
bool const has_ac_time = version == "0.2.0" or version == "0.2.3";
bool const nlink_affects_blocks =
not(version.starts_with("0.2.") or version.starts_with("0.3.") or
version.starts_with("0.4."));
auto const expected_blocks =
nlink_affects_blocks and enable_nlink ? 2747 : 10614;
ASSERT_EQ(0, fs.check(reader::filesystem_check_level::FULL));
vfs_stat vfsbuf;
fs.statvfs(&vfsbuf);
EXPECT_EQ(1, vfsbuf.bsize);
EXPECT_EQ(1, vfsbuf.frsize);
EXPECT_EQ(expected_blocks, vfsbuf.blocks);
EXPECT_EQ(33 + 3 * has_devices, vfsbuf.files);
EXPECT_TRUE(vfsbuf.readonly);
EXPECT_GT(vfsbuf.namemax, 0);
auto json = fs.serialize_metadata_as_json(true);
EXPECT_GT(json.size(), 1000) << json;
std::ostringstream dumpss;
fs.dump(dumpss, {.features = reader::fsinfo_features::all()});
EXPECT_GT(dumpss.str().size(), 1000) << dumpss.str();
auto entry = fs.find("/format.sh");
ASSERT_TRUE(entry);
auto iv = entry->inode();
auto st = fs.getattr(iv);
EXPECT_EQ(94, st.size());
EXPECT_EQ(S_IFREG | 0755, st.mode());
EXPECT_EQ(1000, st.uid());
EXPECT_EQ(100, st.gid());
EXPECT_EQ(1606256045, st.mtime());
if (has_ac_time) {
EXPECT_EQ(1616013831, st.atime());
EXPECT_EQ(1616013816, st.ctime());
}
EXPECT_TRUE(fs.access(iv, R_OK, 1000, 0));
auto inode = fs.open(iv);
EXPECT_GE(inode, 0);
std::error_code ec;
std::vector<char> buf(st.size());
auto rv = fs.read(inode, &buf[0], st.size(), ec);
EXPECT_FALSE(ec);
EXPECT_EQ(rv, st.size());
EXPECT_EQ(format_sh, std::string(buf.begin(), buf.end()));
entry = fs.find("/foo/bad");
ASSERT_TRUE(entry);
iv = entry->inode();
auto link = fs.readlink(iv, reader::readlink_mode::raw);
EXPECT_EQ(link, "../foo");
entry = fs.find(0, "foo");
ASSERT_TRUE(entry);
iv = entry->inode();
auto dir = fs.opendir(iv);
ASSERT_TRUE(dir);
EXPECT_EQ(6 + has_devices, fs.dirsize(*dir));
std::vector<std::string> names;
for (size_t i = 0; i < fs.dirsize(*dir); ++i) {
auto r = fs.readdir(*dir, i);
ASSERT_TRUE(r);
names.emplace_back(r->name());
}
std::vector<std::string> expected{
".", "..", "1", "bad", "bar", "bla.sh",
};
if (has_devices) {
expected.push_back("pipe");
}
EXPECT_EQ(expected, names);
std::map<std::string, file_stat> ref_entries{
{"", make_stat(posix_file_type::directory, 0755, 8)},
{"bench.sh", make_stat(posix_file_type::regular, 0644, 1517)},
{"dev", make_stat(posix_file_type::directory, 0755, 2)},
{"dev/null", make_stat(posix_file_type::character, 0666, 0)},
{"dev/zero", make_stat(posix_file_type::character, 0666, 0)},
{"empty", make_stat(posix_file_type::directory, 0755, 1)},
{"empty/alsoempty", make_stat(posix_file_type::directory, 0755, 0)},
{"foo", make_stat(posix_file_type::directory, 0755, 5)},
{"foo/1", make_stat(posix_file_type::directory, 0755, 2)},
{"foo/1/2", make_stat(posix_file_type::directory, 0755, 2)},
{"foo/1/2/3", make_stat(posix_file_type::directory, 0755, 3)},
{"foo/1/2/3/4", make_stat(posix_file_type::directory, 0755, 2)},
{"foo/1/2/3/4/5", make_stat(posix_file_type::directory, 0755, 2)},
{"foo/1/2/3/4/5/6", make_stat(posix_file_type::directory, 0755, 1)},
{"foo/1/2/3/4/5/6/7", make_stat(posix_file_type::directory, 0755, 1)},
{"foo/1/2/3/4/5/6/7/8", make_stat(posix_file_type::directory, 0755, 1)},
{"foo/1/2/3/4/5/6/7/8/9",
make_stat(posix_file_type::directory, 0755, 13)},
{"foo/1/2/3/4/5/6/7/8/9/a", make_stat(posix_file_type::regular, 0644, 2)},
{"foo/1/2/3/4/5/6/7/8/9/b", make_stat(posix_file_type::regular, 0644, 2)},
{"foo/1/2/3/4/5/6/7/8/9/blubb",
make_stat(posix_file_type::regular, 0644, 1517)},
{"foo/1/2/3/4/5/6/7/8/9/c", make_stat(posix_file_type::regular, 0644, 2)},
{"foo/1/2/3/4/5/6/7/8/9/d", make_stat(posix_file_type::regular, 0644, 2)},
{"foo/1/2/3/4/5/6/7/8/9/e", make_stat(posix_file_type::regular, 0644, 2)},
{"foo/1/2/3/4/5/6/7/8/9/f", make_stat(posix_file_type::regular, 0644, 2)},
{"foo/1/2/3/4/5/6/7/8/9/g", make_stat(posix_file_type::regular, 0644, 2)},
{"foo/1/2/3/4/5/6/7/8/9/h", make_stat(posix_file_type::regular, 0644, 2)},
{"foo/1/2/3/4/5/6/7/8/9/i", make_stat(posix_file_type::regular, 0644, 2)},
{"foo/1/2/3/4/5/6/7/8/9/j", make_stat(posix_file_type::regular, 0644, 2)},
{"foo/1/2/3/4/5/6/7/8/9/k", make_stat(posix_file_type::regular, 0644, 2)},
{"foo/1/2/3/4/5/6/7/8/9/l", make_stat(posix_file_type::regular, 0644, 2)},
{"foo/1/2/3/4/5/z", make_stat(posix_file_type::regular, 0644, 1517)},
{"foo/1/2/3/4/y", make_stat(posix_file_type::regular, 0644, 1517)},
{"foo/1/2/3/copy.sh", make_stat(posix_file_type::regular, 0755, 94)},
{"foo/1/2/3/x", make_stat(posix_file_type::regular, 0644, 1517)},
{"foo/1/2/xxx.sh", make_stat(posix_file_type::regular, 0755, 94)},
{"foo/1/fmt.sh", make_stat(posix_file_type::regular, 0755, 94)},
{"foo/bad", make_stat(posix_file_type::symlink, 0777, 6)},
{"foo/bar", make_stat(posix_file_type::regular, 0644, 0)},
{"foo/bla.sh", make_stat(posix_file_type::regular, 0644, 1517)},
{"foo/pipe", make_stat(posix_file_type::fifo, 0644, 0)},
{"foobar", make_stat(posix_file_type::symlink, 0777, 7)},
{"format.sh", make_stat(posix_file_type::regular, 0755, 94)},
{"perl-exec.sh", make_stat(posix_file_type::regular, 0644, 87)},
{"test.py", make_stat(posix_file_type::regular, 0644, 1012)},
};
if (!has_devices) {
for (auto special : {"dev/null", "dev/zero", "foo/pipe"}) {
ref_entries.erase(special);
}
auto& dev = ref_entries["dev"];
auto& foo = ref_entries["foo"];
dev.set_size(dev.size() - 2);
foo.set_size(foo.size() - 1);
}
enum class walk_mode { normal, data_order, custom };
for (auto mode :
{walk_mode::normal, walk_mode::data_order, walk_mode::custom}) {
std::map<std::string, file_stat> entries;
std::vector<int> inodes;
std::vector<int> first_blocks;
auto cb = [&](auto e) {
auto iv = e.inode();
auto stbuf = fs.getattr(iv);
inodes.push_back(stbuf.ino());
EXPECT_TRUE(entries.emplace(e.unix_path(), stbuf).second);
if (iv.is_regular_file()) {
auto i = fs.get_inode_info(iv);
if (!i["chunks"].empty()) {
first_blocks.push_back(i["chunks"][0]["block"].template get<int>());
}
}
};
switch (mode) {
case walk_mode::normal:
fs.walk(cb);
break;
case walk_mode::data_order:
fs.walk_data_order(cb);
break;
case walk_mode::custom:
walk_tree(fs, cb);
break;
}
EXPECT_EQ(entries.size(), ref_entries.size());
for (auto const& [p, st] : entries) {
auto it = ref_entries.find(p);
EXPECT_TRUE(it != ref_entries.end()) << p;
if (it != ref_entries.end()) {
EXPECT_EQ(it->second.mode(), st.mode()) << p;
if (st.type() == posix_file_type::character) {
EXPECT_EQ(0, st.uid()) << p;
EXPECT_EQ(0, st.gid()) << p;
} else {
EXPECT_EQ(1000, st.uid()) << p;
EXPECT_EQ(100, st.gid()) << p;
}
EXPECT_EQ(it->second.size(), st.size()) << p;
}
}
EXPECT_EQ(24, first_blocks.size());
switch (mode) {
case walk_mode::normal:
case walk_mode::custom:
EXPECT_FALSE(std::is_sorted(first_blocks.begin(), first_blocks.end()));
break;
case walk_mode::data_order:
EXPECT_TRUE(std::is_sorted(first_blocks.begin(), first_blocks.end()));
break;
}
}
#ifndef DWARFS_FILESYSTEM_EXTRACTOR_NO_OPEN_FORMAT
test::os_access_mock os;
utility::filesystem_extractor ext(lgr, os);
std::ostringstream oss;
EXPECT_NO_THROW(ext.open_stream(oss, "mtree"));
EXPECT_NO_THROW(ext.extract(fs));
EXPECT_NO_THROW(ext.close());
ref_entries.erase("");
auto mtree = test::parse_mtree(oss.str());
for (auto [path, kv] : mtree) {
auto name = path.substr(2);
auto ri = ref_entries.find(name);
EXPECT_FALSE(ri == ref_entries.end());
if (ri != ref_entries.end()) {
auto const& st = ri->second;
EXPECT_EQ(kv["mode"], fmt::format("{0:o}", st.mode() & 0777));
EXPECT_EQ(std::stoi(kv["uid"]), kv["type"] == "char" ? 0 : 1000);
EXPECT_EQ(std::stoi(kv["gid"]), kv["type"] == "char" ? 0 : 100);
if (kv["type"] == "file") {
EXPECT_EQ(std::stoi(kv["size"]), st.size());
}
}
}
EXPECT_EQ(ref_entries.size(), mtree.size());
#endif
std::map<std::string, std::vector<std::string>> testdirs{
{"empty", {"empty/alsoempty"}},
{"empty/alsoempty", {}},
{"foo/1/2/3/4/5", {"foo/1/2/3/4/5/6", "foo/1/2/3/4/5/z"}},
{"foo/1/2/3/4", {"foo/1/2/3/4/5", "foo/1/2/3/4/y"}},
{"foo/1/2/3", {"foo/1/2/3/4", "foo/1/2/3/copy.sh", "foo/1/2/3/x"}},
{"",
{"bench.sh", "dev", "empty", "foo", "foobar", "format.sh",
"perl-exec.sh", "test.py"}},
};
for (auto const& [td, expected] : testdirs) {
auto entry = fs.find(td);
ASSERT_TRUE(entry) << td;
auto dir = fs.opendir(entry->inode());
ASSERT_TRUE(dir) << td;
std::vector<std::string> paths;
for (auto const& dev : *dir) {
paths.emplace_back(dev.unix_path());
}
EXPECT_EQ(expected, paths) << td;
paths.clear();
for (auto it = dir->begin(); it != dir->end();) {
paths.emplace_back((it++)->unix_path());
}
EXPECT_EQ(expected, paths) << td;
}
{
auto dev = fs.find("foo/1/2/3/4/5/6/7/8/9/j");
ASSERT_TRUE(dev);
EXPECT_EQ("j", dev->name());
EXPECT_FALSE(dev->is_root());
EXPECT_TRUE(dev->inode().is_regular_file());
dev = dev->parent();
EXPECT_EQ("9", dev->name());
EXPECT_EQ("foo/1/2/3/4/5/6/7/8/9", dev->unix_path());
EXPECT_FALSE(dev->is_root());
EXPECT_TRUE(dev->inode().is_directory());
dev = dev->parent()->parent()->parent();
EXPECT_EQ("6", dev->name());
EXPECT_FALSE(dev->is_root());
EXPECT_TRUE(dev->inode().is_directory());
dev = dev->parent()->parent()->parent()->parent()->parent()->parent();
EXPECT_EQ("foo", dev->name());
EXPECT_FALSE(dev->is_root());
EXPECT_TRUE(dev->inode().is_directory());
dev = dev->parent();
ASSERT_TRUE(dev);
EXPECT_EQ("", dev->name());
EXPECT_TRUE(dev->is_root());
EXPECT_TRUE(dev->inode().is_directory());
EXPECT_FALSE(dev->parent());
}
}
auto get_image_path(std::string const& version) {
return test_dir / "compat" / fmt::format("compat-v{}.dwarfs", version);
}
} // namespace
class compat_metadata : public testing::TestWithParam<std::string> {};
void check_dynamic(std::string const& version,
reader::filesystem_v2 const& fs) {
auto meta = fs.metadata_as_json();
nlohmann::json ref;
if (version.starts_with("0.2.")) {
ref = nlohmann::json::parse(reference_v0_2);
} else {
ref = nlohmann::json::parse(reference_v0_4);
}
remove_inode_numbers(ref);
remove_inode_numbers(meta);
EXPECT_EQ(ref, meta) << nlohmann::json::diff(ref, meta).dump(2);
}
TEST_P(compat_metadata, backwards_compat) {
auto version = GetParam();
auto filename = get_image_path(version);
test::test_logger lgr;
test::os_access_mock os;
reader::filesystem_v2 fs(lgr, os, std::make_shared<mmap>(filename));
check_dynamic(version, fs);
}
INSTANTIATE_TEST_SUITE_P(dwarfs_compat, compat_metadata,
::testing::ValuesIn(versions));
class compat_filesystem
: public testing::TestWithParam<std::tuple<std::string, bool>> {};
TEST_P(compat_filesystem, backwards_compat) {
auto [version, enable_nlink] = GetParam();
test::test_logger lgr;
test::os_access_mock os;
auto filename = get_image_path(version);
reader::filesystem_options opts;
opts.metadata.enable_nlink = enable_nlink;
opts.metadata.check_consistency = true;
{
reader::filesystem_v2 fs(lgr, os, std::make_shared<mmap>(filename), opts);
check_compat(lgr, fs, version, enable_nlink);
}
opts.image_offset = reader::filesystem_options::IMAGE_OFFSET_AUTO;
std::string fsdata;
ASSERT_TRUE(folly::readFile(filename.string().c_str(), fsdata));
for (auto const& hdr : headers) {
reader::filesystem_v2 fs(
lgr, os, std::make_shared<test::mmap_mock>(hdr + fsdata), opts);
check_compat(lgr, fs, version, enable_nlink);
}
if (version != "0.2.0" and version != "0.2.3") {
for (auto const& hdr : headers_v2) {
reader::filesystem_v2 fs(
lgr, os, std::make_shared<test::mmap_mock>(hdr + fsdata), opts);
check_compat(lgr, fs, version, enable_nlink);
}
}
}
INSTANTIATE_TEST_SUITE_P(dwarfs_compat, compat_filesystem,
::testing::Combine(::testing::ValuesIn(versions),
::testing::Bool()));
class rewrite
: public testing::TestWithParam<std::tuple<std::string, bool, bool>> {};
TEST_P(rewrite, filesystem_rewrite) {
auto [version, recompress_block, recompress_metadata] = GetParam();
test::test_logger lgr;
test::os_access_mock os;
auto filename = get_image_path(version);
utility::rewrite_options opts;
opts.recompress_block = recompress_block;
opts.recompress_metadata = recompress_metadata;
thread_pool pool(lgr, os, "rewriter", 2);
block_compressor bc("null");
writer::writer_progress prog;
std::ostringstream rewritten, idss;
auto rewrite_fs = [&](auto& fsw, auto const& mm) {
reader::filesystem_options fsopts;
fsopts.image_offset = reader::filesystem_options::IMAGE_OFFSET_AUTO;
reader::filesystem_v2 fs(lgr, os, mm, fsopts);
writer::filesystem_block_category_resolver resolver(
fs.get_all_block_categories());
utility::rewrite_filesystem(lgr, fs, fsw, resolver, opts);
};
{
writer::filesystem_writer fsw(rewritten, lgr, pool, prog);
fsw.add_default_compressor(bc);
auto mm = std::make_shared<mmap>(filename);
EXPECT_NO_THROW(reader::filesystem_v2::identify(lgr, os, mm, idss));
EXPECT_FALSE(reader::filesystem_v2::header(mm));
rewrite_fs(fsw, mm);
}
{
auto mm = std::make_shared<test::mmap_mock>(rewritten.str());
EXPECT_NO_THROW(reader::filesystem_v2::identify(lgr, os, mm, idss));
EXPECT_FALSE(reader::filesystem_v2::header(mm));
reader::filesystem_v2 fs(lgr, os, mm);
check_dynamic(version, fs);
}
rewritten.str(std::string());
rewritten.clear();
{
std::istringstream hdr_iss(format_sh);
writer::filesystem_writer_options fsw_opts;
writer::filesystem_writer fsw(rewritten, lgr, pool, prog, fsw_opts,
&hdr_iss);
fsw.add_default_compressor(bc);
rewrite_fs(fsw, std::make_shared<mmap>(filename));
}
{
auto mm = std::make_shared<test::mmap_mock>(rewritten.str());
EXPECT_NO_THROW(reader::filesystem_v2::identify(
lgr, os, mm, idss, 0, 1, false,
reader::filesystem_options::IMAGE_OFFSET_AUTO));
auto hdr = reader::filesystem_v2::header(mm);
ASSERT_TRUE(hdr) << folly::hexDump(rewritten.str().data(),
rewritten.str().size());
EXPECT_EQ(format_sh, std::string(reinterpret_cast<char const*>(hdr->data()),
hdr->size()));
reader::filesystem_options fsopts;
fsopts.image_offset = reader::filesystem_options::IMAGE_OFFSET_AUTO;
reader::filesystem_v2 fs(lgr, os, mm, fsopts);
check_dynamic(version, fs);
}
std::ostringstream rewritten2;
{
std::istringstream hdr_iss("D");
writer::filesystem_writer_options fsw_opts;
writer::filesystem_writer fsw(rewritten2, lgr, pool, prog, fsw_opts,
&hdr_iss);
fsw.add_default_compressor(bc);
rewrite_fs(fsw, std::make_shared<test::mmap_mock>(rewritten.str()));
}
{
auto mm = std::make_shared<test::mmap_mock>(rewritten2.str());
auto hdr = reader::filesystem_v2::header(mm);
ASSERT_TRUE(hdr) << folly::hexDump(rewritten2.str().data(),
rewritten2.str().size());
EXPECT_EQ("D", std::string(reinterpret_cast<char const*>(hdr->data()),
hdr->size()));
}
std::ostringstream rewritten3;
{
writer::filesystem_writer fsw(rewritten3, lgr, pool, prog);
fsw.add_default_compressor(bc);
rewrite_fs(fsw, std::make_shared<test::mmap_mock>(rewritten2.str()));
}
{
auto mm = std::make_shared<test::mmap_mock>(rewritten3.str());
auto hdr = reader::filesystem_v2::header(mm);
ASSERT_TRUE(hdr) << folly::hexDump(rewritten3.str().data(),
rewritten3.str().size());
EXPECT_EQ("D", std::string(reinterpret_cast<char const*>(hdr->data()),
hdr->size()));
}
std::ostringstream rewritten4;
{
writer::filesystem_writer_options fsw_opts;
fsw_opts.remove_header = true;
writer::filesystem_writer fsw(rewritten4, lgr, pool, prog, fsw_opts);
fsw.add_default_compressor(bc);
rewrite_fs(fsw, std::make_shared<test::mmap_mock>(rewritten3.str()));
}
{
auto mm = std::make_shared<test::mmap_mock>(rewritten4.str());
EXPECT_NO_THROW(reader::filesystem_v2::identify(lgr, os, mm, idss));
EXPECT_FALSE(reader::filesystem_v2::header(mm))
<< folly::hexDump(rewritten4.str().data(), rewritten4.str().size());
reader::filesystem_v2 fs(lgr, os, mm);
check_dynamic(version, fs);
}
std::ostringstream rewritten5;
{
writer::filesystem_writer_options fsw_opts;
fsw_opts.no_section_index = true;
writer::filesystem_writer fsw(rewritten5, lgr, pool, prog, fsw_opts);
fsw.add_default_compressor(bc);
rewrite_fs(fsw, std::make_shared<test::mmap_mock>(rewritten4.str()));
}
{
auto mm = std::make_shared<test::mmap_mock>(rewritten5.str());
EXPECT_NO_THROW(reader::filesystem_v2::identify(lgr, os, mm, idss));
EXPECT_FALSE(reader::filesystem_v2::header(mm))
<< folly::hexDump(rewritten5.str().data(), rewritten5.str().size());
reader::filesystem_v2 fs(lgr, os, mm);
check_dynamic(version, fs);
}
}
INSTANTIATE_TEST_SUITE_P(dwarfs_compat, rewrite,
::testing::Combine(::testing::ValuesIn(versions),
::testing::Bool(),
::testing::Bool()));
class set_uidgid_test : public testing::TestWithParam<char const*> {};
TEST_P(set_uidgid_test, read_legacy_image) {
auto image = test_dir / "compat" / GetParam();
test::test_logger lgr;
test::os_access_mock os;
reader::filesystem_v2 fs(lgr, os, std::make_shared<mmap>(image));
ASSERT_EQ(0, fs.check(reader::filesystem_check_level::FULL));
for (auto path : {"/dwarfs", "/dwarfs/version.h"}) {
auto v = fs.find(path);
ASSERT_TRUE(v) << path;
auto iv = v->inode();
EXPECT_EQ(33333, iv.getuid()) << path;
EXPECT_EQ(44444, iv.getgid()) << path;
auto st = fs.getattr(iv);
EXPECT_EQ(33333, st.uid()) << path;
EXPECT_EQ(44444, st.gid()) << path;
}
}
namespace {
std::array legacy_images{
"setuidgid-v0.4.1.dwarfs",
"setuidgid-v0.5.6.dwarfs",
};
} // namespace
INSTANTIATE_TEST_SUITE_P(dwarfs_compat, set_uidgid_test,
::testing::ValuesIn(legacy_images));
Reference in New Issue View Git Blame Copy Permalink