From c1fbe90953626f6b96213038b817bdae53a93f84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Mocquillon?= Date: Thu, 2 Mar 2023 16:49:09 +0100 Subject: [PATCH 01/11] Handle GNRL case of ba2 format --- apps/bsatool/bsatool.cpp | 18 +- apps/launcher/datafilespage.cpp | 2 +- apps/niftest/niftest.cpp | 3 +- components/CMakeLists.txt | 2 +- components/bsa/ba2gnrlfile.cpp | 270 +++++++++++++++++++++++++++ components/bsa/ba2gnrlfile.hpp | 55 ++++++ components/bsa/bsa_file.cpp | 51 +++++ components/bsa/bsa_file.hpp | 11 ++ components/bsa/compressedbsafile.cpp | 37 ---- components/bsa/compressedbsafile.hpp | 10 - components/vfs/registerarchives.cpp | 2 +- 11 files changed, 405 insertions(+), 56 deletions(-) create mode 100644 components/bsa/ba2gnrlfile.cpp create mode 100644 components/bsa/ba2gnrlfile.hpp diff --git a/apps/bsatool/bsatool.cpp b/apps/bsatool/bsatool.cpp index d2189a96e0..95920a7862 100644 --- a/apps/bsatool/bsatool.cpp +++ b/apps/bsatool/bsatool.cpp @@ -6,6 +6,7 @@ #include +#include #include #include #include @@ -318,12 +319,19 @@ int main(int argc, char** argv) // Open file - Bsa::BsaVersion bsaVersion = Bsa::CompressedBSAFile::detectVersion(info.filename); + Bsa::BsaVersion bsaVersion = Bsa::BSAFile::detectVersion(info.filename); - if (bsaVersion == Bsa::BSAVER_COMPRESSED) - return call(info); - else - return call(info); + switch (bsaVersion) + { + case Bsa::BSAVER_COMPRESSED: + return call(info); + case Bsa::BSAVER_BA2_GNRL: + return call(info); + case Bsa::BSAVER_UNCOMPRESSED: + return call(info); + default: + throw std::runtime_error("Unrecognised BSA archive"); + } } catch (std::exception& e) { diff --git a/apps/launcher/datafilespage.cpp b/apps/launcher/datafilespage.cpp index aed3e2adf1..2bb31eaf71 100644 --- a/apps/launcher/datafilespage.cpp +++ b/apps/launcher/datafilespage.cpp @@ -725,7 +725,7 @@ void Launcher::DataFilesPage::addArchivesFromDir(const QString& path) for (const auto& fileinfo : dir.entryInfoList()) { const auto absPath = fileinfo.absoluteFilePath(); - if (Bsa::CompressedBSAFile::detectVersion(Files::pathFromQString(absPath)) == Bsa::BSAVER_UNKNOWN) + if (Bsa::BSAFile::detectVersion(Files::pathFromQString(absPath)) == Bsa::BSAVER_UNKNOWN) continue; const auto fileName = fileinfo.fileName(); diff --git a/apps/niftest/niftest.cpp b/apps/niftest/niftest.cpp index 21f40de86f..5f9bfe3235 100644 --- a/apps/niftest/niftest.cpp +++ b/apps/niftest/niftest.cpp @@ -43,7 +43,7 @@ bool isBSA(const std::filesystem::path& filename) std::unique_ptr makeBsaArchive(const std::filesystem::path& path) { - switch (Bsa::CompressedBSAFile::detectVersion(path)) + switch (Bsa::BSAFile::detectVersion(path)) { case Bsa::BSAVER_UNKNOWN: std::cerr << '"' << path << "\" is unknown BSA archive" << std::endl; @@ -51,6 +51,7 @@ std::unique_ptr makeBsaArchive(const std::filesystem::path& path) case Bsa::BSAVER_UNCOMPRESSED: return std::make_unique(path); case Bsa::BSAVER_COMPRESSED: + case Bsa::BSAVER_BA2_GNRL: return std::make_unique(path); } diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index b9704787d1..933e32d733 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -46,7 +46,7 @@ add_component_dir (settings ) add_component_dir (bsa - bsa_file compressedbsafile + bsa_file compressedbsafile ba2gnrlfile ) add_component_dir (vfs diff --git a/components/bsa/ba2gnrlfile.cpp b/components/bsa/ba2gnrlfile.cpp new file mode 100644 index 0000000000..bb5155610c --- /dev/null +++ b/components/bsa/ba2gnrlfile.cpp @@ -0,0 +1,270 @@ +#include "ba2gnrlfile.hpp" + +#include +#include +#include + +#include + +#include +#include + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4706) +#include +#pragma warning(pop) +#else +#include +#endif + +#include +#include +#include +#include +#include +#include + +namespace Bsa +{ + // special marker for invalid records, + // equal to max uint32_t value + const uint32_t sInvalidOffset = std::numeric_limits::max(); + + BA2GNRLFile::FileRecord::FileRecord() + : size(0) + , offset(sInvalidOffset) + { + } + + bool BA2GNRLFile::FileRecord::isValid() const + { + return offset != sInvalidOffset; + } + + BA2GNRLFile::BA2GNRLFile() {} + + BA2GNRLFile::~BA2GNRLFile() = default; + + void BA2GNRLFile::loadFiles(uint32_t fileCount, std::istream& in) + { + mFiles.clear(); + mFiles.reserve(fileCount); + for (uint32_t i = 0; i < fileCount; ++i) + { + uint32_t nameHash, extHash, dirHash; + in.read(reinterpret_cast(&nameHash), sizeof(uint32_t)); + in.read(reinterpret_cast(&extHash), sizeof(uint32_t)); + in.read(reinterpret_cast(&dirHash), sizeof(uint32_t)); + + FileRecord file; + uint32_t unknown; + in.read(reinterpret_cast(&unknown), sizeof(uint32_t)); + in.read(reinterpret_cast(&file.offset), sizeof(int64_t)); + in.read(reinterpret_cast(&file.packedSize), sizeof(uint32_t)); + in.read(reinterpret_cast(&file.size), sizeof(uint32_t)); + + uint32_t baadfood; + in.read(reinterpret_cast(&baadfood), sizeof(uint32_t)); + if (baadfood != 0xBAADF00D) + fail("Corrupted BSA"); + + mFolders[dirHash][{ nameHash, extHash }] = file; + + FileStruct fileStruct{}; + fileStruct.fileSize = file.size; + fileStruct.offset = file.offset; + mFiles.push_back(fileStruct); + } + } + + /// Read header information from the input source + void BA2GNRLFile::readHeader() + { + assert(!mIsLoaded); + + std::ifstream input(mFilepath, std::ios_base::binary); + + // Total archive size + std::streamoff fsize = 0; + if (input.seekg(0, std::ios_base::end)) + { + fsize = input.tellg(); + input.seekg(0); + } + + if (fsize < 24) // header is 24 bytes + fail("File too small to be a valid BSA archive"); + + // Get essential header numbers + uint32_t type, fileCount; + uint64_t fileTableOffset; + { + uint32_t header[4]; + input.read(reinterpret_cast(header), 16); + input.read(reinterpret_cast(&fileTableOffset), 8); + + if (header[0] == 0x00415342) /*"BSA\x00"*/ + fail("Unrecognized compressed BSA format"); + mVersion = header[1]; + if (mVersion != 0x01 /*F04*/) + fail("Unrecognized compressed BSA version"); + + type = header[2]; + fileCount = header[3]; + } + + if (type == ESM::fourCC("GNRL")) + loadFiles(fileCount, input); + else + fail("Unrecognized ba2 version type"); + + // Read the string table + input.seekg(fileTableOffset); + for (uint32_t i = 0; i < fileCount; ++i) + { + std::vector fileName; + uint16_t fileNameSize; + input.read(reinterpret_cast(&fileNameSize), sizeof(uint16_t)); + fileName.resize(fileNameSize); + input.read(fileName.data(), fileName.size()); + fileName.push_back('\0'); + mFileNames.push_back(fileName); + mFiles[i].setNameInfos(0, &mFileNames.back()); + } + + mIsLoaded = true; + } + + BA2GNRLFile::FileRecord BA2GNRLFile::getFileRecord(const std::string& str) const + { + for (const auto c : str) + { + if (((static_cast(c) >> 7U) & 1U) != 0U) + { + fail("File record " + str + " contains unicode characters, refusing to load."); + } + } + +#ifdef _WIN32 + const auto& path = str; +#else + // Force-convert the path into something UNIX can handle first + // to make sure std::filesystem::path doesn't think the entire path is the filename on Linux + // and subsequently purge it to determine the file folder. + std::string path = str; + std::replace(path.begin(), path.end(), '\\', '/'); +#endif + + const auto p = std::filesystem::path{ path }; // Purposefully damage Unicode strings. + const auto fileName = Misc::StringUtils::lowerCase(p.stem().string()); + const auto ext = Misc::StringUtils::lowerCase(p.extension().string()); // Purposefully damage Unicode strings. + const auto folder = Misc::StringUtils::lowerCase(p.parent_path().string()); + + uint32_t folderHash = generateHash(folder); + auto it = mFolders.find(folderHash); + if (it == mFolders.end()) + return FileRecord(); // folder not found, return default which has offset of sInvalidOffset + + uint32_t fileHash = generateHash(fileName); + uint32_t extHash = *reinterpret_cast(ext.data() + 1); + auto iter = it->second.find({ fileHash, extHash }); + if (iter == it->second.end()) + return FileRecord(); // file not found, return default which has offset of sInvalidOffset + return iter->second; + } + + Files::IStreamPtr BA2GNRLFile::getFile(const FileStruct* file) + { + FileRecord fileRec = getFileRecord(file->name()); + if (!fileRec.isValid()) + { + fail("File not found: " + std::string(file->name())); + } + return getFile(fileRec); + } + + void BA2GNRLFile::addFile(const std::string& filename, std::istream& file) + { + assert(false); // not implemented yet + fail("Add file is not implemented for compressed BSA: " + filename); + } + + Files::IStreamPtr BA2GNRLFile::getFile(const char* file) + { + FileRecord fileRec = getFileRecord(file); + if (!fileRec.isValid()) + { + fail("File not found: " + std::string(file)); + } + return getFile(fileRec); + } + + Files::IStreamPtr BA2GNRLFile::getFile(const FileRecord& fileRecord) + { + Files::IStreamPtr streamPtr + = Files::openConstrainedFileStream(mFilepath, fileRecord.offset, fileRecord.packedSize); + std::istream* fileStream = streamPtr.get(); + uint32_t uncompressedSize = fileRecord.size; + auto memoryStreamPtr = std::make_unique(uncompressedSize); + if (fileRecord.packedSize != 0) + { + boost::iostreams::filtering_streambuf inputStreamBuf; + inputStreamBuf.push(boost::iostreams::zlib_decompressor()); + inputStreamBuf.push(*fileStream); + + boost::iostreams::basic_array_sink sr(memoryStreamPtr->getRawData(), uncompressedSize); + boost::iostreams::copy(inputStreamBuf, sr); + } + else + { + fileStream->read(memoryStreamPtr->getRawData(), fileRecord.size); + } + return std::make_unique>(std::move(memoryStreamPtr)); + } + + constexpr const uint32_t crc32table[256] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, + 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, + 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 0xc8d75180, + 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, + 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, + 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, + 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 0x5edef90e, 0x29d9c998, 0xb0d09822, + 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, + 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, + 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, 0xcb61b38c, + 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, + 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, + 0xf6b9265b, 0x6fb077e1, 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, + 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, + 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d }; + + uint32_t BA2GNRLFile::generateHash(const std::string& name) + { + uint32_t result = 0; + for (auto c : name) + { + if (uint8_t(c) > 127) + continue; + if (c == '/') + c = '\\'; + result = (result >> 8) ^ crc32table[(result ^ (unsigned)(c)) & 0xFF]; + } + return result; + } + +} // namespace Bsa diff --git a/components/bsa/ba2gnrlfile.hpp b/components/bsa/ba2gnrlfile.hpp new file mode 100644 index 0000000000..3a2b005f15 --- /dev/null +++ b/components/bsa/ba2gnrlfile.hpp @@ -0,0 +1,55 @@ +#ifndef BSA_BA2_GNRL_FILE_H +#define BSA_BA2_GNRL_FILE_H + +#include +#include + +#include +#include + +namespace Bsa +{ + class BA2GNRLFile : private BSAFile + { + private: + struct FileRecord + { + FileRecord(); + uint32_t size; + uint32_t offset; + uint32_t packedSize = 0; + bool isValid() const; + }; + + uint32_t mVersion{ 0u }; + + using FolderRecord = std::map, FileRecord>; + std::map mFolders; + + std::list> mFileNames; + + FileRecord getFileRecord(const std::string& str) const; + + static uint32_t generateHash(const std::string& name); + Files::IStreamPtr getFile(const FileRecord& fileRecord); + + void loadFiles(uint32_t fileCount, std::istream& in); + + public: + using BSAFile::getFilename; + using BSAFile::getList; + using BSAFile::open; + + BA2GNRLFile(); + virtual ~BA2GNRLFile(); + + /// Read header information from the input source + void readHeader() override; + + Files::IStreamPtr getFile(const char* filePath); + Files::IStreamPtr getFile(const FileStruct* fileStruct); + void addFile(const std::string& filename, std::istream& file); + }; +} + +#endif diff --git a/components/bsa/bsa_file.cpp b/components/bsa/bsa_file.cpp index aae63746d5..e4766d31d3 100644 --- a/components/bsa/bsa_file.cpp +++ b/components/bsa/bsa_file.cpp @@ -23,6 +23,7 @@ #include "bsa_file.hpp" +#include #include #include @@ -319,3 +320,53 @@ void Bsa::BSAFile::addFile(const std::string& filename, std::istream& file) file.seekg(0, std::ios::beg); stream << file.rdbuf(); } + +BsaVersion Bsa::BSAFile::detectVersion(const std::filesystem::path& filePath) +{ + std::ifstream input(filePath, std::ios_base::binary); + + // Total archive size + std::streamoff fsize = 0; + if (input.seekg(0, std::ios_base::end)) + { + fsize = input.tellg(); + input.seekg(0); + } + + if (fsize < 12) + { + return BSAVER_UNKNOWN; + } + + // Get essential header numbers + + // First 12 bytes + uint32_t head[3]; + + input.read(reinterpret_cast(head), 12); + + if (head[0] == static_cast(BSAVER_UNCOMPRESSED)) + { + return BSAVER_UNCOMPRESSED; + } + + if (head[0] == static_cast(BSAVER_COMPRESSED)) + { + if (head[1] == static_cast(0x01)) + { + if (head[2] == ESM::fourCC("GNRL")) + return BSAVER_BA2_GNRL; + return BSAVER_UNKNOWN; + } + return BSAVER_COMPRESSED; + } + + if (head[0] == ESM::fourCC("BTDX")) + { + if (head[2] == ESM::fourCC("GNRL")) + return BSAVER_BA2_GNRL; + return BSAVER_UNKNOWN; + } + + return BSAVER_UNKNOWN; +} diff --git a/components/bsa/bsa_file.hpp b/components/bsa/bsa_file.hpp index bc77819567..a014082ccc 100644 --- a/components/bsa/bsa_file.hpp +++ b/components/bsa/bsa_file.hpp @@ -35,6 +35,14 @@ namespace Bsa { + enum BsaVersion + { + BSAVER_UNKNOWN = 0x0, + BSAVER_UNCOMPRESSED = 0x100, + BSAVER_COMPRESSED = 0x415342, // B, S, A, + BSAVER_BA2_GNRL, + }; + /** This class is used to read "Bethesda Archive Files", or BSAs. */ @@ -138,6 +146,9 @@ namespace Bsa { return Files::pathToUnicodeString(mFilepath); } + + // checks version of BSA from file header + static BsaVersion detectVersion(const std::filesystem::path& filePath); }; } diff --git a/components/bsa/compressedbsafile.cpp b/components/bsa/compressedbsafile.cpp index 47026b55f0..c07480a613 100644 --- a/components/bsa/compressedbsafile.cpp +++ b/components/bsa/compressedbsafile.cpp @@ -432,43 +432,6 @@ namespace Bsa return std::make_unique>(std::move(memoryStreamPtr)); } - BsaVersion CompressedBSAFile::detectVersion(const std::filesystem::path& filePath) - { - std::ifstream input(filePath, std::ios_base::binary); - - // Total archive size - std::streamoff fsize = 0; - if (input.seekg(0, std::ios_base::end)) - { - fsize = input.tellg(); - input.seekg(0); - } - - if (fsize < 12) - { - return BSAVER_UNKNOWN; - } - - // Get essential header numbers - - // First 12 bytes - uint32_t head[3]; - - input.read(reinterpret_cast(head), 12); - - if (head[0] == static_cast(BSAVER_UNCOMPRESSED)) - { - return BSAVER_UNCOMPRESSED; - } - - if (head[0] == static_cast(BSAVER_COMPRESSED)) - { - return BSAVER_COMPRESSED; - } - - return BSAVER_UNKNOWN; - } - // mFiles used by OpenMW expects uncompressed sizes void CompressedBSAFile::convertCompressedSizesToUncompressed() { diff --git a/components/bsa/compressedbsafile.hpp b/components/bsa/compressedbsafile.hpp index 9bafedeee9..cecfaf6866 100644 --- a/components/bsa/compressedbsafile.hpp +++ b/components/bsa/compressedbsafile.hpp @@ -33,13 +33,6 @@ namespace Bsa { - enum BsaVersion - { - BSAVER_UNKNOWN = 0x0, - BSAVER_UNCOMPRESSED = 0x100, - BSAVER_COMPRESSED = 0x415342 // B, S, A - }; - class CompressedBSAFile : private BSAFile { private: @@ -95,9 +88,6 @@ namespace Bsa CompressedBSAFile(); virtual ~CompressedBSAFile(); - // checks version of BSA from file header - static BsaVersion detectVersion(const std::filesystem::path& filePath); - /// Read header information from the input source void readHeader() override; diff --git a/components/vfs/registerarchives.cpp b/components/vfs/registerarchives.cpp index a095d9a3a9..4897c45fa2 100644 --- a/components/vfs/registerarchives.cpp +++ b/components/vfs/registerarchives.cpp @@ -25,7 +25,7 @@ namespace VFS // Last BSA has the highest priority const auto archivePath = collections.getPath(*archive); Log(Debug::Info) << "Adding BSA archive " << archivePath; - Bsa::BsaVersion bsaVersion = Bsa::CompressedBSAFile::detectVersion(archivePath); + Bsa::BsaVersion bsaVersion = Bsa::BSAFile::detectVersion(archivePath); if (bsaVersion == Bsa::BSAVER_COMPRESSED) vfs->addArchive(std::make_unique(archivePath)); From 974799c01133971eb68c050034f18c8845d75f9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Mocquillon?= Date: Thu, 2 Mar 2023 16:51:04 +0100 Subject: [PATCH 02/11] Handle DX10 case of ba2 format --- apps/bsatool/bsatool.cpp | 3 + apps/niftest/niftest.cpp | 1 + components/CMakeLists.txt | 2 +- components/bsa/ba2dx10file.cpp | 706 +++++++++++++++++++++++++++++++++ components/bsa/ba2dx10file.hpp | 70 ++++ components/bsa/bsa_file.cpp | 4 + components/bsa/bsa_file.hpp | 1 + 7 files changed, 786 insertions(+), 1 deletion(-) create mode 100644 components/bsa/ba2dx10file.cpp create mode 100644 components/bsa/ba2dx10file.hpp diff --git a/apps/bsatool/bsatool.cpp b/apps/bsatool/bsatool.cpp index 95920a7862..e2029f3245 100644 --- a/apps/bsatool/bsatool.cpp +++ b/apps/bsatool/bsatool.cpp @@ -6,6 +6,7 @@ #include +#include #include #include #include @@ -327,6 +328,8 @@ int main(int argc, char** argv) return call(info); case Bsa::BSAVER_BA2_GNRL: return call(info); + case Bsa::BSAVER_BA2_DX10: + return call(info); case Bsa::BSAVER_UNCOMPRESSED: return call(info); default: diff --git a/apps/niftest/niftest.cpp b/apps/niftest/niftest.cpp index 5f9bfe3235..169c7fd934 100644 --- a/apps/niftest/niftest.cpp +++ b/apps/niftest/niftest.cpp @@ -52,6 +52,7 @@ std::unique_ptr makeBsaArchive(const std::filesystem::path& path) return std::make_unique(path); case Bsa::BSAVER_COMPRESSED: case Bsa::BSAVER_BA2_GNRL: + case Bsa::BSAVER_BA2_DX10: return std::make_unique(path); } diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index 933e32d733..fc918ecc76 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -46,7 +46,7 @@ add_component_dir (settings ) add_component_dir (bsa - bsa_file compressedbsafile ba2gnrlfile + bsa_file compressedbsafile ba2gnrlfile ba2dx10file ) add_component_dir (vfs diff --git a/components/bsa/ba2dx10file.cpp b/components/bsa/ba2dx10file.cpp new file mode 100644 index 0000000000..432bac73eb --- /dev/null +++ b/components/bsa/ba2dx10file.cpp @@ -0,0 +1,706 @@ +#include "ba2dx10file.hpp" + +#include +#include +#include + +#include + +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4706) +#include +#pragma warning(pop) +#else +#include +#endif + +#include +#include +#include +#include +#include +#include + +namespace Bsa +{ + BA2DX10File::BA2DX10File() {} + + BA2DX10File::~BA2DX10File() = default; + + void BA2DX10File::loadFiles(uint32_t fileCount, std::istream& in) + { + mFiles.clear(); + mFiles.reserve(fileCount); + for (uint32_t i = 0; i < fileCount; ++i) + { + uint32_t nameHash, extHash, dirHash; + in.read(reinterpret_cast(&nameHash), sizeof(uint32_t)); + in.read(reinterpret_cast(&extHash), sizeof(uint32_t)); + in.read(reinterpret_cast(&dirHash), sizeof(uint32_t)); + + FileRecord file; + uint32_t unknown; + in.read(reinterpret_cast(&unknown), sizeof(uint8_t)); + + uint8_t nbChunks; + in.read(reinterpret_cast(&nbChunks), sizeof(uint8_t)); + + file.texturesChunks.resize(nbChunks); + + uint16_t chunkHeaderSize; + in.read(reinterpret_cast(&chunkHeaderSize), sizeof(uint16_t)); + if (chunkHeaderSize != 24) + fail("Corrupted BSA"); + + in.read(reinterpret_cast(&file.height), sizeof(uint16_t)); + in.read(reinterpret_cast(&file.width), sizeof(uint16_t)); + in.read(reinterpret_cast(&file.numMips), sizeof(uint8_t)); + in.read(reinterpret_cast(&file.DXGIFormat), sizeof(uint8_t)); + in.read(reinterpret_cast(&file.cubeMaps), sizeof(uint16_t)); + for (auto& texture : file.texturesChunks) + { + in.read(reinterpret_cast(&texture.offset), sizeof(int64_t)); + in.read(reinterpret_cast(&texture.packedSize), sizeof(uint32_t)); + in.read(reinterpret_cast(&texture.size), sizeof(uint32_t)); + in.read(reinterpret_cast(&texture.startMip), sizeof(uint16_t)); + in.read(reinterpret_cast(&texture.endMip), sizeof(uint16_t)); + uint32_t baadfood; + in.read(reinterpret_cast(&baadfood), sizeof(uint32_t)); + if (baadfood != 0xBAADF00D) + fail("Corrupted BSA"); + } + + mFolders[dirHash][{ nameHash, extHash }] = file; + + FileStruct fileStruct{}; + mFiles.push_back(fileStruct); + } + } + + /// Read header information from the input source + void BA2DX10File::readHeader() + { + assert(!mIsLoaded); + + std::ifstream input(mFilepath, std::ios_base::binary); + + // Total archive size + std::streamoff fsize = 0; + if (input.seekg(0, std::ios_base::end)) + { + fsize = input.tellg(); + input.seekg(0); + } + + if (fsize < 24) // header is 24 bytes + fail("File too small to be a valid BSA archive"); + + // Get essential header numbers + uint32_t type, fileCount; + uint64_t fileTableOffset; + { + uint32_t header[4]; + input.read(reinterpret_cast(header), 16); + input.read(reinterpret_cast(&fileTableOffset), 8); + + if (header[0] == 0x00415342) /*"BSA\x00"*/ + fail("Unrecognized compressed BSA format"); + mVersion = header[1]; + if (mVersion != 0x01 /*F04*/) + fail("Unrecognized compressed BSA version"); + + type = header[2]; + fileCount = header[3]; + } + + if (type == ESM::fourCC("DX10")) + loadFiles(fileCount, input); + else + fail("Unrecognized ba2 version type"); + + // Read the string table + input.seekg(fileTableOffset); + for (uint32_t i = 0; i < fileCount; ++i) + { + std::vector fileName; + uint16_t fileNameSize; + input.read(reinterpret_cast(&fileNameSize), sizeof(uint16_t)); + fileName.resize(fileNameSize); + input.read(fileName.data(), fileName.size()); + fileName.push_back('\0'); + mFileNames.push_back(fileName); + mFiles[i].setNameInfos(0, &mFileNames.back()); + } + + mIsLoaded = true; + } + + BA2DX10File::FileRecord BA2DX10File::getFileRecord(const std::string& str) const + { + for (const auto c : str) + { + if (((static_cast(c) >> 7U) & 1U) != 0U) + { + fail("File record " + str + " contains unicode characters, refusing to load."); + } + } + +#ifdef _WIN32 + const auto& path = str; +#else + // Force-convert the path into something UNIX can handle first + // to make sure std::filesystem::path doesn't think the entire path is the filename on Linux + // and subsequently purge it to determine the file folder. + std::string path = str; + std::replace(path.begin(), path.end(), '\\', '/'); +#endif + + const auto p = std::filesystem::path{ path }; // Purposefully damage Unicode strings. + const auto fileName = Misc::StringUtils::lowerCase(p.stem().string()); + const auto ext = Misc::StringUtils::lowerCase(p.extension().string()); // Purposefully damage Unicode strings. + const auto folder = Misc::StringUtils::lowerCase(p.parent_path().string()); + + uint32_t folderHash = generateHash(folder); + auto it = mFolders.find(folderHash); + if (it == mFolders.end()) + return FileRecord(); // folder not found, return default which has offset of sInvalidOffset + + uint32_t fileHash = generateHash(fileName); + uint32_t extHash = *reinterpret_cast(ext.data() + 1); + auto iter = it->second.find({ fileHash, extHash }); + if (iter == it->second.end()) + return FileRecord(); // file not found, return default which has offset of sInvalidOffset + return iter->second; + } + +#pragma pack(push) +#pragma pack(1) + struct DDSHeader + { + uint32_t size = 0; + uint32_t flags = 0; + uint32_t height = 0; + uint32_t width = 0; + uint32_t pitchOrLinearSize = 0; + uint32_t depth = 0; + uint32_t mipMapCount = 0; + uint32_t reserved1[11] = {}; + struct + { + uint32_t size = 0; + uint32_t flags = 0; + uint32_t fourCC = 0; + uint32_t RGBBitCount = 0; + uint32_t RBitMask = 0; + uint32_t GBitMask = 0; + uint32_t BBitMask = 0; + uint32_t ABitMask = 0; + } ddspf; + uint32_t caps = 0; + uint32_t caps2 = 0; + uint32_t caps3 = 0; + uint32_t caps4 = 0; + uint32_t reserved2 = 0; + }; + + struct DDSHeaderDX10 : DDSHeader + { + int32_t dxgiFormat = 0; + uint32_t resourceDimension = 0; + uint32_t miscFlags = 0; + uint32_t arraySize = 0; + uint32_t miscFlags2 = 0; + }; +#pragma pack(pop) + + Files::IStreamPtr BA2DX10File::getFile(const FileStruct* file) + { + FileRecord fileRec = getFileRecord(file->name()); + return getFile(fileRec); + } + + void BA2DX10File::addFile(const std::string& filename, std::istream& file) + { + assert(false); // not implemented yet + fail("Add file is not implemented for compressed BSA: " + filename); + } + + Files::IStreamPtr BA2DX10File::getFile(const char* file) + { + FileRecord fileRec = getFileRecord(file); + return getFile(fileRec); + } + + constexpr const uint32_t DDSD_CAPS = 0x00000001; + constexpr const uint32_t DDSD_HEIGHT = 0x00000002; + constexpr const uint32_t DDSD_WIDTH = 0x00000004; + constexpr const uint32_t DDSD_PITCH = 0x00000008; + constexpr const uint32_t DDSD_PIXELFORMAT = 0x00001000; + constexpr const uint32_t DDSD_MIPMAPCOUNT = 0x00020000; + constexpr const uint32_t DDSD_LINEARSIZE = 0x00080000; + + constexpr const uint32_t DDSCAPS_COMPLEX = 0x00000008; + constexpr const uint32_t DDSCAPS_TEXTURE = 0x00001000; + constexpr const uint32_t DDSCAPS_MIPMAP = 0x00400000; + + constexpr const uint32_t DDSCAPS2_CUBEMAP = 0x00000200; + constexpr const uint32_t DDSCAPS2_POSITIVEX = 0x00000400; + constexpr const uint32_t DDSCAPS2_NEGATIVEX = 0x00000800; + constexpr const uint32_t DDSCAPS2_POSITIVEY = 0x00001000; + constexpr const uint32_t DDSCAPS2_NEGATIVEY = 0x00002000; + constexpr const uint32_t DDSCAPS2_POSITIVEZ = 0x00004000; + constexpr const uint32_t DDSCAPS2_NEGATIVEZ = 0x00008000; + + constexpr const uint32_t DDPF_ALPHAPIXELS = 0x00000001; + constexpr const uint32_t DDPF_ALPHA = 0x00000002; + constexpr const uint32_t DDPF_FOURCC = 0x00000004; + constexpr const uint32_t DDPF_RGB = 0x00000040; + constexpr const uint32_t DDPF_LUMINANCE = 0x00020000; + + constexpr const uint32_t DDS_DIMENSION_TEXTURE2D = 0x00000003; + constexpr const uint32_t DDS_RESOURCE_MISC_TEXTURECUBE = 0x00000004; + + enum DXGI : uint8_t + { + DXGI_FORMAT_UNKNOWN = 0, + DXGI_FORMAT_R32G32B32A32_TYPELESS, + DXGI_FORMAT_R32G32B32A32_FLOAT, + DXGI_FORMAT_R32G32B32A32_UINT, + DXGI_FORMAT_R32G32B32A32_SINT, + DXGI_FORMAT_R32G32B32_TYPELESS, + DXGI_FORMAT_R32G32B32_FLOAT, + DXGI_FORMAT_R32G32B32_UINT, + DXGI_FORMAT_R32G32B32_SINT, + DXGI_FORMAT_R16G16B16A16_TYPELESS, + DXGI_FORMAT_R16G16B16A16_FLOAT, + DXGI_FORMAT_R16G16B16A16_UNORM, + DXGI_FORMAT_R16G16B16A16_UINT, + DXGI_FORMAT_R16G16B16A16_SNORM, + DXGI_FORMAT_R16G16B16A16_SINT, + DXGI_FORMAT_R32G32_TYPELESS, + DXGI_FORMAT_R32G32_FLOAT, + DXGI_FORMAT_R32G32_UINT, + DXGI_FORMAT_R32G32_SINT, + DXGI_FORMAT_R32G8X24_TYPELESS, + DXGI_FORMAT_D32_FLOAT_S8X24_UINT, + DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS, + DXGI_FORMAT_X32_TYPELESS_G8X24_UINT, + DXGI_FORMAT_R10G10B10A2_TYPELESS, + DXGI_FORMAT_R10G10B10A2_UNORM, + DXGI_FORMAT_R10G10B10A2_UINT, + DXGI_FORMAT_R11G11B10_FLOAT, + DXGI_FORMAT_R8G8B8A8_TYPELESS, + DXGI_FORMAT_R8G8B8A8_UNORM, + DXGI_FORMAT_R8G8B8A8_UNORM_SRGB, + DXGI_FORMAT_R8G8B8A8_UINT, + DXGI_FORMAT_R8G8B8A8_SNORM, + DXGI_FORMAT_R8G8B8A8_SINT, + DXGI_FORMAT_R16G16_TYPELESS, + DXGI_FORMAT_R16G16_FLOAT, + DXGI_FORMAT_R16G16_UNORM, + DXGI_FORMAT_R16G16_UINT, + DXGI_FORMAT_R16G16_SNORM, + DXGI_FORMAT_R16G16_SINT, + DXGI_FORMAT_R32_TYPELESS, + DXGI_FORMAT_D32_FLOAT, + DXGI_FORMAT_R32_FLOAT, + DXGI_FORMAT_R32_UINT, + DXGI_FORMAT_R32_SINT, + DXGI_FORMAT_R24G8_TYPELESS, + DXGI_FORMAT_D24_UNORM_S8_UINT, + DXGI_FORMAT_R24_UNORM_X8_TYPELESS, + DXGI_FORMAT_X24_TYPELESS_G8_UINT, + DXGI_FORMAT_R8G8_TYPELESS, + DXGI_FORMAT_R8G8_UNORM, + DXGI_FORMAT_R8G8_UINT, + DXGI_FORMAT_R8G8_SNORM, + DXGI_FORMAT_R8G8_SINT, + DXGI_FORMAT_R16_TYPELESS, + DXGI_FORMAT_R16_FLOAT, + DXGI_FORMAT_D16_UNORM, + DXGI_FORMAT_R16_UNORM, + DXGI_FORMAT_R16_UINT, + DXGI_FORMAT_R16_SNORM, + DXGI_FORMAT_R16_SINT, + DXGI_FORMAT_R8_TYPELESS, + DXGI_FORMAT_R8_UNORM, + DXGI_FORMAT_R8_UINT, + DXGI_FORMAT_R8_SNORM, + DXGI_FORMAT_R8_SINT, + DXGI_FORMAT_A8_UNORM, + DXGI_FORMAT_R1_UNORM, + DXGI_FORMAT_R9G9B9E5_SHAREDEXP, + DXGI_FORMAT_R8G8_B8G8_UNORM, + DXGI_FORMAT_G8R8_G8B8_UNORM, + DXGI_FORMAT_BC1_TYPELESS, + DXGI_FORMAT_BC1_UNORM, + DXGI_FORMAT_BC1_UNORM_SRGB, + DXGI_FORMAT_BC2_TYPELESS, + DXGI_FORMAT_BC2_UNORM, + DXGI_FORMAT_BC2_UNORM_SRGB, + DXGI_FORMAT_BC3_TYPELESS, + DXGI_FORMAT_BC3_UNORM, + DXGI_FORMAT_BC3_UNORM_SRGB, + DXGI_FORMAT_BC4_TYPELESS, + DXGI_FORMAT_BC4_UNORM, + DXGI_FORMAT_BC4_SNORM, + DXGI_FORMAT_BC5_TYPELESS, + DXGI_FORMAT_BC5_UNORM, + DXGI_FORMAT_BC5_SNORM, + DXGI_FORMAT_B5G6R5_UNORM, + DXGI_FORMAT_B5G5R5A1_UNORM, + DXGI_FORMAT_B8G8R8A8_UNORM, + DXGI_FORMAT_B8G8R8X8_UNORM, + DXGI_FORMAT_R10G10B10_XR_BIAS_A2_UNORM, + DXGI_FORMAT_B8G8R8A8_TYPELESS, + DXGI_FORMAT_B8G8R8A8_UNORM_SRGB, + DXGI_FORMAT_B8G8R8X8_TYPELESS, + DXGI_FORMAT_B8G8R8X8_UNORM_SRGB, + DXGI_FORMAT_BC6H_TYPELESS, + DXGI_FORMAT_BC6H_UF16, + DXGI_FORMAT_BC6H_SF16, + DXGI_FORMAT_BC7_TYPELESS, + DXGI_FORMAT_BC7_UNORM, + DXGI_FORMAT_BC7_UNORM_SRGB, + DXGI_FORMAT_AYUV, + DXGI_FORMAT_Y410, + DXGI_FORMAT_Y416, + DXGI_FORMAT_NV12, + DXGI_FORMAT_P010, + DXGI_FORMAT_P016, + DXGI_FORMAT_420_OPAQUE, + DXGI_FORMAT_YUY2, + DXGI_FORMAT_Y210, + DXGI_FORMAT_Y216, + DXGI_FORMAT_NV11, + DXGI_FORMAT_AI44, + DXGI_FORMAT_IA44, + DXGI_FORMAT_P8, + DXGI_FORMAT_A8P8, + DXGI_FORMAT_B4G4R4A4_UNORM, + DXGI_FORMAT_P208, + DXGI_FORMAT_V208, + DXGI_FORMAT_V408 + }; + + Files::IStreamPtr BA2DX10File::getFile(const FileRecord& fileRecord) + { + DDSHeaderDX10 header; + header.size = sizeof(DDSHeader); + header.width = fileRecord.width; + header.height = fileRecord.height; + header.flags = DDSD_CAPS | DDSD_PIXELFORMAT | DDSD_WIDTH | DDSD_HEIGHT | DDSD_MIPMAPCOUNT; + header.caps = DDSCAPS_TEXTURE; + header.mipMapCount = fileRecord.numMips; + if (header.mipMapCount > 1) + header.caps = header.caps | DDSCAPS_MIPMAP | DDSCAPS_COMPLEX; + header.depth = 1; + + header.resourceDimension = DDS_DIMENSION_TEXTURE2D; + header.arraySize = 1; + + if (fileRecord.cubeMaps == 2049) + { + header.caps = header.caps | DDSCAPS_COMPLEX; + header.caps2 = DDSCAPS2_CUBEMAP | DDSCAPS2_POSITIVEX | DDSCAPS2_NEGATIVEX | DDSCAPS2_POSITIVEY + | DDSCAPS2_NEGATIVEY | DDSCAPS2_POSITIVEZ | DDSCAPS2_NEGATIVEZ; + header.miscFlags = DDS_RESOURCE_MISC_TEXTURECUBE; + } + header.ddspf.size = sizeof(header.ddspf); + switch (DXGI(fileRecord.DXGIFormat)) + { + case DXGI_FORMAT_BC1_UNORM: + { + header.flags = header.flags | DDSD_LINEARSIZE; + header.ddspf.flags = DDPF_FOURCC; + header.ddspf.fourCC = ESM::fourCC("DXT1"); + header.pitchOrLinearSize = fileRecord.width * fileRecord.height / 2; + break; + } + case DXGI_FORMAT_BC2_UNORM: + { + header.flags = header.flags | DDSD_LINEARSIZE; + header.ddspf.flags = DDPF_FOURCC; + header.ddspf.fourCC = ESM::fourCC("DXT3"); + header.pitchOrLinearSize = fileRecord.width * fileRecord.height; + break; + } + case DXGI_FORMAT_BC3_UNORM: + { + header.flags = header.flags | DDSD_LINEARSIZE; + header.ddspf.flags = DDPF_FOURCC; + header.ddspf.fourCC = ESM::fourCC("DXT5"); + header.pitchOrLinearSize = fileRecord.width * fileRecord.height; + break; + } + case DXGI_FORMAT_BC4_SNORM: + { + header.flags = header.flags | DDSD_LINEARSIZE; + header.ddspf.flags = DDPF_FOURCC; + header.ddspf.fourCC = ESM::fourCC("BC4S"); + header.pitchOrLinearSize = fileRecord.width * fileRecord.height / 2; + break; + } + case DXGI_FORMAT_BC4_UNORM: + { + header.flags = header.flags | DDSD_LINEARSIZE; + header.ddspf.flags = DDPF_FOURCC; + header.ddspf.fourCC = ESM::fourCC("BC4U"); + header.pitchOrLinearSize = fileRecord.width * fileRecord.height / 2; + break; + } + case DXGI_FORMAT_BC5_SNORM: + { + header.flags = header.flags | DDSD_LINEARSIZE; + header.ddspf.flags = DDPF_FOURCC; + header.ddspf.fourCC = ESM::fourCC("BC5S"); + header.pitchOrLinearSize = fileRecord.width * fileRecord.height; + break; + } + case DXGI_FORMAT_BC5_UNORM: + { + header.flags = header.flags | DDSD_LINEARSIZE; + header.ddspf.flags = DDPF_FOURCC; + header.ddspf.fourCC = ESM::fourCC("BC5U"); + header.pitchOrLinearSize = fileRecord.width * fileRecord.height; + break; + } + case DXGI_FORMAT_BC1_UNORM_SRGB: + { + header.flags = header.flags | DDSD_LINEARSIZE; + header.ddspf.flags = DDPF_FOURCC; + header.ddspf.fourCC = ESM::fourCC("DX10"); + header.dxgiFormat = int32_t(fileRecord.DXGIFormat); + header.pitchOrLinearSize = fileRecord.width * fileRecord.height / 2; + break; + } + case DXGI_FORMAT_BC2_UNORM_SRGB: + case DXGI_FORMAT_BC3_UNORM_SRGB: + case DXGI_FORMAT_BC6H_UF16: + case DXGI_FORMAT_BC6H_SF16: + case DXGI_FORMAT_BC7_UNORM: + case DXGI_FORMAT_BC7_UNORM_SRGB: + { + header.flags = header.flags | DDSD_LINEARSIZE; + header.ddspf.flags = DDPF_FOURCC; + header.ddspf.fourCC = ESM::fourCC("DX10"); + header.dxgiFormat = int32_t(fileRecord.DXGIFormat); + header.pitchOrLinearSize = fileRecord.width * fileRecord.height; + break; + } + case DXGI_FORMAT_B8G8R8A8_UNORM_SRGB: + case DXGI_FORMAT_B8G8R8X8_UNORM_SRGB: + case DXGI_FORMAT_R8G8B8A8_SINT: + case DXGI_FORMAT_R8G8B8A8_UINT: + case DXGI_FORMAT_R8G8B8A8_UNORM_SRGB: + { + header.flags = header.flags | DDSD_PITCH; + header.ddspf.flags = DDPF_FOURCC; + header.ddspf.fourCC = ESM::fourCC("DX10"); + header.dxgiFormat = int32_t(fileRecord.DXGIFormat); + header.pitchOrLinearSize = fileRecord.width * 4; + break; + } + case DXGI_FORMAT_R8G8_SINT: + case DXGI_FORMAT_R8G8_UINT: + { + header.flags = header.flags | DDSD_PITCH; + header.ddspf.flags = DDPF_FOURCC; + header.ddspf.fourCC = ESM::fourCC("DX10"); + header.dxgiFormat = int32_t(fileRecord.DXGIFormat); + header.pitchOrLinearSize = fileRecord.width * 2; + break; + } + case DXGI_FORMAT_R8_SINT: + case DXGI_FORMAT_R8_SNORM: + case DXGI_FORMAT_R8_UINT: + { + header.flags = header.flags | DDSD_PITCH; + header.ddspf.flags = DDPF_FOURCC; + header.ddspf.fourCC = ESM::fourCC("DX10"); + header.dxgiFormat = int32_t(fileRecord.DXGIFormat); + header.pitchOrLinearSize = fileRecord.width; + break; + } + case DXGI_FORMAT_R8G8B8A8_UNORM: + { + header.flags = header.flags | DDSD_PITCH; + header.ddspf.flags = DDPF_RGB | DDPF_ALPHAPIXELS; + header.ddspf.RGBBitCount = 32; + header.ddspf.RBitMask = 0x000000FF; + header.ddspf.GBitMask = 0x0000FF00; + header.ddspf.BBitMask = 0x00FF0000; + header.ddspf.ABitMask = 0xFF000000; + header.pitchOrLinearSize = fileRecord.width * 4; + break; + } + case DXGI_FORMAT_B8G8R8A8_UNORM: + { + header.flags = header.flags | DDSD_PITCH; + header.ddspf.flags = DDPF_RGB | DDPF_ALPHAPIXELS; + header.ddspf.RGBBitCount = 32; + header.ddspf.RBitMask = 0x00FF0000; + header.ddspf.GBitMask = 0x0000FF00; + header.ddspf.BBitMask = 0x000000FF; + header.ddspf.ABitMask = 0xFF000000; + header.pitchOrLinearSize = fileRecord.width * 4; + break; + } + case DXGI_FORMAT_B8G8R8X8_UNORM: + { + header.flags = header.flags | DDSD_PITCH; + header.ddspf.flags = DDPF_RGB; + header.ddspf.RGBBitCount = 32; + header.ddspf.RBitMask = 0x00FF0000; + header.ddspf.GBitMask = 0x0000FF00; + header.ddspf.BBitMask = 0x000000FF; + header.pitchOrLinearSize = fileRecord.width * 4; + break; + } + case DXGI_FORMAT_B5G6R5_UNORM: + { + header.flags = header.flags | DDSD_PITCH; + header.ddspf.flags = DDPF_RGB; + header.ddspf.RGBBitCount = 16; + header.ddspf.RBitMask = 0x0000F800; + header.ddspf.GBitMask = 0x000007E0; + header.ddspf.BBitMask = 0x0000001F; + header.pitchOrLinearSize = fileRecord.width * 2; + break; + } + case DXGI_FORMAT_B5G5R5A1_UNORM: + { + header.flags = header.flags | DDSD_PITCH; + header.ddspf.flags = DDPF_RGB | DDPF_ALPHAPIXELS; + header.ddspf.RGBBitCount = 16; + header.ddspf.RBitMask = 0x00007C00; + header.ddspf.GBitMask = 0x000003E0; + header.ddspf.BBitMask = 0x0000001F; + header.ddspf.ABitMask = 0x00008000; + header.pitchOrLinearSize = fileRecord.width * 2; + break; + } + case DXGI_FORMAT_R8G8_UNORM: + { + header.flags = header.flags | DDSD_PITCH; + header.ddspf.flags = DDPF_LUMINANCE | DDPF_ALPHAPIXELS; + header.ddspf.RGBBitCount = 16; + header.ddspf.RBitMask = 0x000000FF; + header.ddspf.ABitMask = 0x0000FF00; + header.pitchOrLinearSize = fileRecord.width * 2; + break; + } + case DXGI_FORMAT_A8_UNORM: + { + header.flags = header.flags | DDSD_PITCH; + header.ddspf.flags = DDPF_ALPHA; + header.ddspf.RGBBitCount = 8; + header.ddspf.ABitMask = 0x000000FF; + header.pitchOrLinearSize = fileRecord.width; + break; + } + case DXGI_FORMAT_R8_UNORM: + { + header.flags = header.flags | DDSD_PITCH; + header.ddspf.flags = DDPF_LUMINANCE; + header.ddspf.RGBBitCount = 8; + header.ddspf.RBitMask = 0x000000FF; + header.pitchOrLinearSize = fileRecord.width; + break; + } + default: + break; + } + + size_t headerSize = (header.ddspf.fourCC == ESM::fourCC("DX10") ? sizeof(DDSHeaderDX10) : sizeof(DDSHeader)); + + size_t textureSize = sizeof(uint32_t) + headerSize; //"DDS " + header + for (const auto& textureChunk : fileRecord.texturesChunks) + textureSize += textureChunk.size; + + auto memoryStreamPtr = std::make_unique(textureSize); + char* buff = memoryStreamPtr->getRawData(); + + uint32_t dds = ESM::fourCC("DDS "); + buff = (char*)std::memcpy(buff, &dds, sizeof(uint32_t)) + sizeof(uint32_t); + std::memcpy(buff, &header, headerSize); + + size_t offset = headerSize; + // append chunks + for (const auto& c : fileRecord.texturesChunks) + { + if (c.packedSize != 0) + { + Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilepath, c.offset, c.packedSize); + std::istream* fileStream = streamPtr.get(); + + boost::iostreams::filtering_streambuf inputStreamBuf; + inputStreamBuf.push(boost::iostreams::zlib_decompressor()); + inputStreamBuf.push(*fileStream); + + boost::iostreams::basic_array_sink sr(memoryStreamPtr->getRawData() + offset, c.size); + boost::iostreams::copy(inputStreamBuf, sr); + } + // uncompressed chunk + else + { + Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilepath, c.offset, c.packedSize); + std::istream* fileStream = streamPtr.get(); + fileStream->read(memoryStreamPtr->getRawData(), c.size); + } + offset += c.size; + } + + return std::make_unique>(std::move(memoryStreamPtr)); + } + + constexpr const uint32_t crc32table[256] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, + 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, + 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 0xc8d75180, + 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, + 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, + 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, + 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 0x5edef90e, 0x29d9c998, 0xb0d09822, + 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, + 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, + 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, 0xcb61b38c, + 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, + 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, + 0xf6b9265b, 0x6fb077e1, 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, + 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, + 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d }; + + uint32_t BA2DX10File::generateHash(const std::string& name) + { + uint32_t result = 0; + for (auto c : name) + { + if (uint8_t(c) > 127) + continue; + if (c == '/') + c = '\\'; + result = (result >> 8) ^ crc32table[(result ^ (unsigned)(c)) & 0xFF]; + } + return result; + } + +} // namespace Bsa diff --git a/components/bsa/ba2dx10file.hpp b/components/bsa/ba2dx10file.hpp new file mode 100644 index 0000000000..422b2317e9 --- /dev/null +++ b/components/bsa/ba2dx10file.hpp @@ -0,0 +1,70 @@ +#ifndef BSA_BA2_DX10_FILE_H +#define BSA_BA2_DX10_FILE_H + +#include +#include + +#include +#include + +namespace Bsa +{ + class BA2DX10File : private BSAFile + { + private: + struct TextureChunkRecord + { + uint32_t size = 0; + uint32_t packedSize = 0; + int64_t offset = 0; + uint16_t startMip = 0; + uint16_t endMip = 0; + }; + + struct FileRecord + { + // DX10 archive format + uint8_t unknownTex = 0; + // ChunkHeaderSize: Word; + uint16_t height = 0; + uint16_t width = 0; + uint8_t numMips = 0; + uint8_t DXGIFormat = 0; + uint16_t cubeMaps = 0; + std::vector texturesChunks; + // + // std::string DXGIFormatName() const; + }; + + uint32_t mVersion{ 0u }; + + using FolderRecord = std::map, FileRecord>; + std::map mFolders; + + std::list> mFileNames; + + FileRecord getFileRecord(const std::string& str) const; + + static uint32_t generateHash(const std::string& name); + Files::IStreamPtr getFile(const FileRecord& fileRecord); + + void loadFiles(uint32_t fileCount, std::istream& in); + + public: + using BSAFile::getFilename; + using BSAFile::getList; + using BSAFile::open; + + BA2DX10File(); + virtual ~BA2DX10File(); + + /// Read header information from the input source + void readHeader() override; + + Files::IStreamPtr getFile(const char* filePath); + Files::IStreamPtr getFile(const FileStruct* fileStruct); + void addFile(const std::string& filename, std::istream& file); + }; +} + +#endif diff --git a/components/bsa/bsa_file.cpp b/components/bsa/bsa_file.cpp index e4766d31d3..ec95e86729 100644 --- a/components/bsa/bsa_file.cpp +++ b/components/bsa/bsa_file.cpp @@ -356,6 +356,8 @@ BsaVersion Bsa::BSAFile::detectVersion(const std::filesystem::path& filePath) { if (head[2] == ESM::fourCC("GNRL")) return BSAVER_BA2_GNRL; + if (head[2] == ESM::fourCC("DX10")) + return BSAVER_BA2_DX10; return BSAVER_UNKNOWN; } return BSAVER_COMPRESSED; @@ -365,6 +367,8 @@ BsaVersion Bsa::BSAFile::detectVersion(const std::filesystem::path& filePath) { if (head[2] == ESM::fourCC("GNRL")) return BSAVER_BA2_GNRL; + if (head[2] == ESM::fourCC("DX10")) + return BSAVER_BA2_DX10; return BSAVER_UNKNOWN; } diff --git a/components/bsa/bsa_file.hpp b/components/bsa/bsa_file.hpp index a014082ccc..db68c0c485 100644 --- a/components/bsa/bsa_file.hpp +++ b/components/bsa/bsa_file.hpp @@ -41,6 +41,7 @@ namespace Bsa BSAVER_UNCOMPRESSED = 0x100, BSAVER_COMPRESSED = 0x415342, // B, S, A, BSAVER_BA2_GNRL, + BSAVER_BA2_DX10 }; /** From d30deef4b51dd818ff077a85a54fe64a81048160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Mocquillon?= Date: Thu, 2 Mar 2023 20:47:53 +0100 Subject: [PATCH 03/11] Use the right archive type to load bsa according to their type --- apps/niftest/niftest.cpp | 8 +- components/vfs/bsaarchive.cpp | 120 ---------------------------- components/vfs/bsaarchive.hpp | 114 +++++++++++++++++--------- components/vfs/registerarchives.cpp | 10 ++- 4 files changed, 91 insertions(+), 161 deletions(-) delete mode 100644 components/vfs/bsaarchive.cpp diff --git a/apps/niftest/niftest.cpp b/apps/niftest/niftest.cpp index 169c7fd934..9bb9b67143 100644 --- a/apps/niftest/niftest.cpp +++ b/apps/niftest/niftest.cpp @@ -48,12 +48,14 @@ std::unique_ptr makeBsaArchive(const std::filesystem::path& path) case Bsa::BSAVER_UNKNOWN: std::cerr << '"' << path << "\" is unknown BSA archive" << std::endl; return nullptr; - case Bsa::BSAVER_UNCOMPRESSED: - return std::make_unique(path); case Bsa::BSAVER_COMPRESSED: + return std::make_unique::type>(path); case Bsa::BSAVER_BA2_GNRL: + return std::make_unique::type>(path); case Bsa::BSAVER_BA2_DX10: - return std::make_unique(path); + return std::make_unique::type>(path); + case Bsa::BSAVER_UNCOMPRESSED: + return std::make_unique::type>(path); } std::cerr << '"' << path << "\" is unsupported BSA archive" << std::endl; diff --git a/components/vfs/bsaarchive.cpp b/components/vfs/bsaarchive.cpp deleted file mode 100644 index 3de393c120..0000000000 --- a/components/vfs/bsaarchive.cpp +++ /dev/null @@ -1,120 +0,0 @@ -#include "bsaarchive.hpp" - -#include -#include - -namespace VFS -{ - - BsaArchive::BsaArchive(const std::filesystem::path& filename) - { - mFile = std::make_unique(); - mFile->open(filename); - - const Bsa::BSAFile::FileList& filelist = mFile->getList(); - for (Bsa::BSAFile::FileList::const_iterator it = filelist.begin(); it != filelist.end(); ++it) - { - mResources.emplace_back(&*it, mFile.get()); - } - } - - BsaArchive::BsaArchive() {} - - BsaArchive::~BsaArchive() {} - - void BsaArchive::listResources(std::map& out, char (*normalize_function)(char)) - { - for (std::vector::iterator it = mResources.begin(); it != mResources.end(); ++it) - { - std::string ent = it->mInfo->name(); - std::transform(ent.begin(), ent.end(), ent.begin(), normalize_function); - - out[ent] = &*it; - } - } - - bool BsaArchive::contains(const std::string& file, char (*normalize_function)(char)) const - { - for (const auto& it : mResources) - { - std::string ent = it.mInfo->name(); - std::transform(ent.begin(), ent.end(), ent.begin(), normalize_function); - if (file == ent) - return true; - } - return false; - } - - std::string BsaArchive::getDescription() const - { - return std::string{ "BSA: " } + mFile->getFilename(); - } - - // ------------------------------------------------------------------------------ - - BsaArchiveFile::BsaArchiveFile(const Bsa::BSAFile::FileStruct* info, Bsa::BSAFile* bsa) - : mInfo(info) - , mFile(bsa) - { - } - - Files::IStreamPtr BsaArchiveFile::open() - { - return mFile->getFile(mInfo); - } - - CompressedBsaArchive::CompressedBsaArchive(const std::filesystem::path& filename) - : Archive() - { - mCompressedFile = std::make_unique(); - mCompressedFile->open(filename); - - const Bsa::BSAFile::FileList& filelist = mCompressedFile->getList(); - for (Bsa::BSAFile::FileList::const_iterator it = filelist.begin(); it != filelist.end(); ++it) - { - mCompressedResources.emplace_back(&*it, mCompressedFile.get()); - } - } - - void CompressedBsaArchive::listResources(std::map& out, char (*normalize_function)(char)) - { - for (std::vector::iterator it = mCompressedResources.begin(); - it != mCompressedResources.end(); ++it) - { - std::string ent = it->mInfo->name(); - std::transform(ent.begin(), ent.end(), ent.begin(), normalize_function); - - out[ent] = &*it; - } - } - - bool CompressedBsaArchive::contains(const std::string& file, char (*normalize_function)(char)) const - { - for (const auto& it : mCompressedResources) - { - std::string ent = it.mInfo->name(); - std::transform(ent.begin(), ent.end(), ent.begin(), normalize_function); - if (file == ent) - return true; - } - return false; - } - - std::string CompressedBsaArchive::getDescription() const - { - return std::string{ "BSA: " } + mCompressedFile->getFilename(); - } - - CompressedBsaArchiveFile::CompressedBsaArchiveFile( - const Bsa::BSAFile::FileStruct* info, Bsa::CompressedBSAFile* bsa) - : mInfo(info) - , mCompressedFile(bsa) - { - } - - Files::IStreamPtr CompressedBsaArchiveFile::open() - { - return mCompressedFile->getFile(mInfo); - } - -} diff --git a/components/vfs/bsaarchive.hpp b/components/vfs/bsaarchive.hpp index 9d1b6123e5..019715518e 100644 --- a/components/vfs/bsaarchive.hpp +++ b/components/vfs/bsaarchive.hpp @@ -3,66 +3,108 @@ #include "archive.hpp" +#include +#include #include #include namespace VFS { + template class BsaArchiveFile : public File { public: - BsaArchiveFile(const Bsa::BSAFile::FileStruct* info, Bsa::BSAFile* bsa); + BsaArchiveFile(const Bsa::BSAFile::FileStruct* info, FileType* bsa) + : mInfo(info) + , mFile(bsa) + { + } - Files::IStreamPtr open() override; + Files::IStreamPtr open() override { return mFile->getFile(mInfo); } std::filesystem::path getPath() override { return mInfo->name(); } const Bsa::BSAFile::FileStruct* mInfo; - Bsa::BSAFile* mFile; - }; - - class CompressedBsaArchiveFile : public File - { - public: - CompressedBsaArchiveFile(const Bsa::BSAFile::FileStruct* info, Bsa::CompressedBSAFile* bsa); - - Files::IStreamPtr open() override; - - std::filesystem::path getPath() override { return mInfo->name(); } - - const Bsa::BSAFile::FileStruct* mInfo; - Bsa::CompressedBSAFile* mCompressedFile; + FileType* mFile; }; + template class BsaArchive : public Archive { public: - BsaArchive(const std::filesystem::path& filename); - BsaArchive(); - virtual ~BsaArchive(); - void listResources(std::map& out, char (*normalize_function)(char)) override; - bool contains(const std::string& file, char (*normalize_function)(char)) const override; - std::string getDescription() const override; + BsaArchive(const std::filesystem::path& filename) + : Archive() + { + mFile = std::make_unique(); + mFile->open(filename); - protected: - std::unique_ptr mFile; - std::vector mResources; - }; + const Bsa::BSAFile::FileList& filelist = mFile->getList(); + for (Bsa::BSAFile::FileList::const_iterator it = filelist.begin(); it != filelist.end(); ++it) + { + mResources.emplace_back(&*it, mFile.get()); + } + } - class CompressedBsaArchive : public Archive - { - public: - CompressedBsaArchive(const std::filesystem::path& filename); - virtual ~CompressedBsaArchive() {} - void listResources(std::map& out, char (*normalize_function)(char)) override; - bool contains(const std::string& file, char (*normalize_function)(char)) const override; - std::string getDescription() const override; + virtual ~BsaArchive() {} + + void listResources(std::map& out, char (*normalize_function)(char)) override + { + for (auto& resource : mResources) + { + std::string ent = resource.mInfo->name(); + std::transform(ent.begin(), ent.end(), ent.begin(), normalize_function); + + out[ent] = &resource; + } + } + + bool contains(const std::string& file, char (*normalize_function)(char)) const override + { + for (const auto& it : mResources) + { + std::string ent = it.mInfo->name(); + std::transform(ent.begin(), ent.end(), ent.begin(), normalize_function); + if (file == ent) + return true; + } + return false; + } + + std::string getDescription() const override { return std::string{ "BSA: " } + mFile->getFilename(); } private: - std::unique_ptr mCompressedFile; - std::vector mCompressedResources; + std::unique_ptr mFile; + std::vector> mResources; }; + template + struct ArchiveSelector + { + }; + + template <> + struct ArchiveSelector + { + using type = BsaArchive; + }; + + template <> + struct ArchiveSelector + { + using type = BsaArchive; + }; + + template <> + struct ArchiveSelector + { + using type = BsaArchive; + }; + + template <> + struct ArchiveSelector + { + using type = BsaArchive; + }; } #endif diff --git a/components/vfs/registerarchives.cpp b/components/vfs/registerarchives.cpp index 4897c45fa2..9dbe878bca 100644 --- a/components/vfs/registerarchives.cpp +++ b/components/vfs/registerarchives.cpp @@ -28,9 +28,15 @@ namespace VFS Bsa::BsaVersion bsaVersion = Bsa::BSAFile::detectVersion(archivePath); if (bsaVersion == Bsa::BSAVER_COMPRESSED) - vfs->addArchive(std::make_unique(archivePath)); + vfs->addArchive(std::make_unique::type>(archivePath)); + else if (bsaVersion == Bsa::BSAVER_BA2_GNRL) + vfs->addArchive(std::make_unique::type>(archivePath)); + else if (bsaVersion == Bsa::BSAVER_BA2_DX10) + vfs->addArchive(std::make_unique::type>(archivePath)); + else if (bsaVersion == Bsa::BSAVER_UNCOMPRESSED) + vfs->addArchive(std::make_unique::type>(archivePath)); else - vfs->addArchive(std::make_unique(archivePath)); + throw std::runtime_error("Unknown archive type '" + *archive + "'"); } else { From 24272d07bfbfb25449202ba0ea618529a7c96cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Mocquillon?= Date: Fri, 3 Mar 2023 13:27:25 +0100 Subject: [PATCH 04/11] Share generateHash used in ba2dx10file and ba2gnrlfile --- components/CMakeLists.txt | 2 +- components/bsa/ba2dx10file.cpp | 45 +------------------------------ components/bsa/ba2dx10file.hpp | 1 - components/bsa/ba2file.cpp | 49 ++++++++++++++++++++++++++++++++++ components/bsa/ba2file.hpp | 11 ++++++++ components/bsa/ba2gnrlfile.cpp | 45 +------------------------------ components/bsa/ba2gnrlfile.hpp | 1 - 7 files changed, 63 insertions(+), 91 deletions(-) create mode 100644 components/bsa/ba2file.cpp create mode 100644 components/bsa/ba2file.hpp diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index fc918ecc76..a78d8c61cb 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -46,7 +46,7 @@ add_component_dir (settings ) add_component_dir (bsa - bsa_file compressedbsafile ba2gnrlfile ba2dx10file + bsa_file compressedbsafile ba2gnrlfile ba2dx10file ba2file memorystream ) add_component_dir (vfs diff --git a/components/bsa/ba2dx10file.cpp b/components/bsa/ba2dx10file.cpp index 432bac73eb..6caf9d6cab 100644 --- a/components/bsa/ba2dx10file.cpp +++ b/components/bsa/ba2dx10file.cpp @@ -20,6 +20,7 @@ #endif #include +#include #include #include #include @@ -659,48 +660,4 @@ namespace Bsa return std::make_unique>(std::move(memoryStreamPtr)); } - constexpr const uint32_t crc32table[256] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, - 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, - 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, - 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, - 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 0xc8d75180, - 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, - 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, - 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, - 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, - 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, - 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, - 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 0x5edef90e, 0x29d9c998, 0xb0d09822, - 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, - 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, - 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, - 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, - 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, 0xcb61b38c, - 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, - 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, - 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, - 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, - 0xf6b9265b, 0x6fb077e1, 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, - 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, - 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, - 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d }; - - uint32_t BA2DX10File::generateHash(const std::string& name) - { - uint32_t result = 0; - for (auto c : name) - { - if (uint8_t(c) > 127) - continue; - if (c == '/') - c = '\\'; - result = (result >> 8) ^ crc32table[(result ^ (unsigned)(c)) & 0xFF]; - } - return result; - } - } // namespace Bsa diff --git a/components/bsa/ba2dx10file.hpp b/components/bsa/ba2dx10file.hpp index 422b2317e9..833d271e0a 100644 --- a/components/bsa/ba2dx10file.hpp +++ b/components/bsa/ba2dx10file.hpp @@ -45,7 +45,6 @@ namespace Bsa FileRecord getFileRecord(const std::string& str) const; - static uint32_t generateHash(const std::string& name); Files::IStreamPtr getFile(const FileRecord& fileRecord); void loadFiles(uint32_t fileCount, std::istream& in); diff --git a/components/bsa/ba2file.cpp b/components/bsa/ba2file.cpp new file mode 100644 index 0000000000..b4fc7f9ec2 --- /dev/null +++ b/components/bsa/ba2file.cpp @@ -0,0 +1,49 @@ +#include "ba2file.hpp" + +namespace Bsa +{ + constexpr const uint32_t crc32table[256] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, + 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, + 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, + 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 0xc8d75180, + 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, + 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, + 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, + 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, + 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, + 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 0x5edef90e, 0x29d9c998, 0xb0d09822, + 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, + 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, + 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, 0xcb61b38c, + 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, + 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, + 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, + 0xf6b9265b, 0x6fb077e1, 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, + 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, + 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, + 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d }; + + uint32_t generateHash(const std::string& name) + { + uint32_t result = 0; + for (auto c : name) + { + if (uint8_t(c) > 127) + continue; + if (c == '/') + c = '\\'; + result = (result >> 8) ^ crc32table[(result ^ (unsigned)(c)) & 0xFF]; + } + return result; + } + +} // namespace Bsa diff --git a/components/bsa/ba2file.hpp b/components/bsa/ba2file.hpp new file mode 100644 index 0000000000..77160737e8 --- /dev/null +++ b/components/bsa/ba2file.hpp @@ -0,0 +1,11 @@ +#ifndef BSA_BA2_FILE_H +#define BSA_BA2_FILE_H + +#include + +namespace Bsa +{ + uint32_t generateHash(const std::string& name); +} + +#endif diff --git a/components/bsa/ba2gnrlfile.cpp b/components/bsa/ba2gnrlfile.cpp index bb5155610c..14eb671f93 100644 --- a/components/bsa/ba2gnrlfile.cpp +++ b/components/bsa/ba2gnrlfile.cpp @@ -19,6 +19,7 @@ #endif #include +#include #include #include #include @@ -223,48 +224,4 @@ namespace Bsa return std::make_unique>(std::move(memoryStreamPtr)); } - constexpr const uint32_t crc32table[256] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, - 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, - 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, - 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, - 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 0xc8d75180, - 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, - 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, - 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, - 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, - 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, - 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, - 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 0x5edef90e, 0x29d9c998, 0xb0d09822, - 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, - 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, - 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, - 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, - 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, 0xcb61b38c, - 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, - 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, - 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, - 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, - 0xf6b9265b, 0x6fb077e1, 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, - 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, - 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, - 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d }; - - uint32_t BA2GNRLFile::generateHash(const std::string& name) - { - uint32_t result = 0; - for (auto c : name) - { - if (uint8_t(c) > 127) - continue; - if (c == '/') - c = '\\'; - result = (result >> 8) ^ crc32table[(result ^ (unsigned)(c)) & 0xFF]; - } - return result; - } - } // namespace Bsa diff --git a/components/bsa/ba2gnrlfile.hpp b/components/bsa/ba2gnrlfile.hpp index 3a2b005f15..8c0333d8d1 100644 --- a/components/bsa/ba2gnrlfile.hpp +++ b/components/bsa/ba2gnrlfile.hpp @@ -30,7 +30,6 @@ namespace Bsa FileRecord getFileRecord(const std::string& str) const; - static uint32_t generateHash(const std::string& name); Files::IStreamPtr getFile(const FileRecord& fileRecord); void loadFiles(uint32_t fileCount, std::istream& in); From d73ab7ef6feb3405b901976093f56520d49fd5cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Mocquillon?= Date: Fri, 3 Mar 2023 13:27:40 +0100 Subject: [PATCH 05/11] Add comments --- components/bsa/ba2gnrlfile.cpp | 3 +-- components/bsa/bsa_file.cpp | 1 + components/bsa/bsa_file.hpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/bsa/ba2gnrlfile.cpp b/components/bsa/ba2gnrlfile.cpp index 14eb671f93..0e4cabd075 100644 --- a/components/bsa/ba2gnrlfile.cpp +++ b/components/bsa/ba2gnrlfile.cpp @@ -29,7 +29,6 @@ namespace Bsa { // special marker for invalid records, - // equal to max uint32_t value const uint32_t sInvalidOffset = std::numeric_limits::max(); BA2GNRLFile::FileRecord::FileRecord() @@ -108,7 +107,7 @@ namespace Bsa if (header[0] == 0x00415342) /*"BSA\x00"*/ fail("Unrecognized compressed BSA format"); mVersion = header[1]; - if (mVersion != 0x01 /*F04*/) + if (mVersion != 0x01 /*FO4*/) fail("Unrecognized compressed BSA version"); type = header[2]; diff --git a/components/bsa/bsa_file.cpp b/components/bsa/bsa_file.cpp index ec95e86729..a8604dfe83 100644 --- a/components/bsa/bsa_file.cpp +++ b/components/bsa/bsa_file.cpp @@ -365,6 +365,7 @@ BsaVersion Bsa::BSAFile::detectVersion(const std::filesystem::path& filePath) if (head[0] == ESM::fourCC("BTDX")) { + // head[1] should contains 0x01 it is checked if we want to open this kind of archive if (head[2] == ESM::fourCC("GNRL")) return BSAVER_BA2_GNRL; if (head[2] == ESM::fourCC("DX10")) diff --git a/components/bsa/bsa_file.hpp b/components/bsa/bsa_file.hpp index db68c0c485..8a953245d2 100644 --- a/components/bsa/bsa_file.hpp +++ b/components/bsa/bsa_file.hpp @@ -40,8 +40,8 @@ namespace Bsa BSAVER_UNKNOWN = 0x0, BSAVER_UNCOMPRESSED = 0x100, BSAVER_COMPRESSED = 0x415342, // B, S, A, - BSAVER_BA2_GNRL, - BSAVER_BA2_DX10 + BSAVER_BA2_GNRL, // used by FO4, BSA which contains files + BSAVER_BA2_DX10 // used by FO4, BSA which contains textures }; /** From 78251a2bb17cb7f130db78834a961bd725271033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Mocquillon?= Date: Fri, 3 Mar 2023 13:29:13 +0100 Subject: [PATCH 06/11] Move outside the condition + fix bug: missing offset --- components/bsa/ba2dx10file.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/components/bsa/ba2dx10file.cpp b/components/bsa/ba2dx10file.cpp index 6caf9d6cab..6557670bee 100644 --- a/components/bsa/ba2dx10file.cpp +++ b/components/bsa/ba2dx10file.cpp @@ -635,11 +635,11 @@ namespace Bsa // append chunks for (const auto& c : fileRecord.texturesChunks) { + Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilepath, c.offset, c.packedSize); + std::istream* fileStream = streamPtr.get(); + if (c.packedSize != 0) { - Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilepath, c.offset, c.packedSize); - std::istream* fileStream = streamPtr.get(); - boost::iostreams::filtering_streambuf inputStreamBuf; inputStreamBuf.push(boost::iostreams::zlib_decompressor()); inputStreamBuf.push(*fileStream); @@ -650,9 +650,7 @@ namespace Bsa // uncompressed chunk else { - Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilepath, c.offset, c.packedSize); - std::istream* fileStream = streamPtr.get(); - fileStream->read(memoryStreamPtr->getRawData(), c.size); + fileStream->read(memoryStreamPtr->getRawData() + offset, c.size); } offset += c.size; } From d2244c8d976f48ea2bce10572e5ee8c4f12b4f47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Mocquillon?= Date: Fri, 3 Mar 2023 20:49:31 +0100 Subject: [PATCH 07/11] Fix detectVersion --- components/bsa/bsa_file.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/components/bsa/bsa_file.cpp b/components/bsa/bsa_file.cpp index a8604dfe83..8fc2d3f8d9 100644 --- a/components/bsa/bsa_file.cpp +++ b/components/bsa/bsa_file.cpp @@ -365,12 +365,14 @@ BsaVersion Bsa::BSAFile::detectVersion(const std::filesystem::path& filePath) if (head[0] == ESM::fourCC("BTDX")) { - // head[1] should contains 0x01 it is checked if we want to open this kind of archive - if (head[2] == ESM::fourCC("GNRL")) - return BSAVER_BA2_GNRL; - if (head[2] == ESM::fourCC("DX10")) - return BSAVER_BA2_DX10; - return BSAVER_UNKNOWN; + if (head[1] == static_cast(0x01)) + { + if (head[2] == ESM::fourCC("GNRL")) + return BSAVER_BA2_GNRL; + if (head[2] == ESM::fourCC("DX10")) + return BSAVER_BA2_DX10; + } + return BSAVER_COMPRESSED; } return BSAVER_UNKNOWN; From 1a7b2cd611cab1fb3e83bfa80ebbd04164fde0a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Mocquillon?= Date: Mon, 6 Mar 2023 15:50:45 +0100 Subject: [PATCH 08/11] Fix remaining bugs --- components/bsa/ba2dx10file.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/components/bsa/ba2dx10file.cpp b/components/bsa/ba2dx10file.cpp index 6557670bee..4ec8359b85 100644 --- a/components/bsa/ba2dx10file.cpp +++ b/components/bsa/ba2dx10file.cpp @@ -45,7 +45,7 @@ namespace Bsa in.read(reinterpret_cast(&dirHash), sizeof(uint32_t)); FileRecord file; - uint32_t unknown; + uint8_t unknown; in.read(reinterpret_cast(&unknown), sizeof(uint8_t)); uint8_t nbChunks; @@ -400,7 +400,7 @@ namespace Bsa header.mipMapCount = fileRecord.numMips; if (header.mipMapCount > 1) header.caps = header.caps | DDSCAPS_MIPMAP | DDSCAPS_COMPLEX; - header.depth = 1; + header.depth = 0; header.resourceDimension = DDS_DIMENSION_TEXTURE2D; header.arraySize = 1; @@ -631,15 +631,15 @@ namespace Bsa buff = (char*)std::memcpy(buff, &dds, sizeof(uint32_t)) + sizeof(uint32_t); std::memcpy(buff, &header, headerSize); - size_t offset = headerSize; + size_t offset = sizeof(uint32_t) + headerSize; // append chunks for (const auto& c : fileRecord.texturesChunks) { - Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilepath, c.offset, c.packedSize); - std::istream* fileStream = streamPtr.get(); - if (c.packedSize != 0) { + Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilepath, c.offset, c.packedSize); + std::istream* fileStream = streamPtr.get(); + boost::iostreams::filtering_streambuf inputStreamBuf; inputStreamBuf.push(boost::iostreams::zlib_decompressor()); inputStreamBuf.push(*fileStream); @@ -650,6 +650,9 @@ namespace Bsa // uncompressed chunk else { + Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilepath, c.offset, c.size); + std::istream* fileStream = streamPtr.get(); + fileStream->read(memoryStreamPtr->getRawData() + offset, c.size); } offset += c.size; From 64ce6eb6bc74b0d0e75e870807d3d506339378bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Mocquillon?= Date: Mon, 6 Mar 2023 21:25:53 +0100 Subject: [PATCH 09/11] Merge the two identical conditions --- components/bsa/bsa_file.cpp | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/components/bsa/bsa_file.cpp b/components/bsa/bsa_file.cpp index 8fc2d3f8d9..b3e24c75ab 100644 --- a/components/bsa/bsa_file.cpp +++ b/components/bsa/bsa_file.cpp @@ -350,7 +350,7 @@ BsaVersion Bsa::BSAFile::detectVersion(const std::filesystem::path& filePath) return BSAVER_UNCOMPRESSED; } - if (head[0] == static_cast(BSAVER_COMPRESSED)) + if (head[0] == static_cast(BSAVER_COMPRESSED) || head[0] == ESM::fourCC("BTDX")) { if (head[1] == static_cast(0x01)) { @@ -363,17 +363,5 @@ BsaVersion Bsa::BSAFile::detectVersion(const std::filesystem::path& filePath) return BSAVER_COMPRESSED; } - if (head[0] == ESM::fourCC("BTDX")) - { - if (head[1] == static_cast(0x01)) - { - if (head[2] == ESM::fourCC("GNRL")) - return BSAVER_BA2_GNRL; - if (head[2] == ESM::fourCC("DX10")) - return BSAVER_BA2_DX10; - } - return BSAVER_COMPRESSED; - } - return BSAVER_UNKNOWN; } From 7f936ad18f9ed2252a063f145cabcefa6c38b607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Mocquillon?= Date: Mon, 6 Mar 2023 21:26:20 +0100 Subject: [PATCH 10/11] Some cleanup on includes --- components/bsa/ba2dx10file.hpp | 8 +++----- components/bsa/ba2gnrlfile.hpp | 3 ++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/components/bsa/ba2dx10file.hpp b/components/bsa/ba2dx10file.hpp index 833d271e0a..398e555d30 100644 --- a/components/bsa/ba2dx10file.hpp +++ b/components/bsa/ba2dx10file.hpp @@ -3,9 +3,11 @@ #include #include +#include +#include +#include #include -#include namespace Bsa { @@ -23,17 +25,13 @@ namespace Bsa struct FileRecord { - // DX10 archive format uint8_t unknownTex = 0; - // ChunkHeaderSize: Word; uint16_t height = 0; uint16_t width = 0; uint8_t numMips = 0; uint8_t DXGIFormat = 0; uint16_t cubeMaps = 0; std::vector texturesChunks; - // - // std::string DXGIFormatName() const; }; uint32_t mVersion{ 0u }; diff --git a/components/bsa/ba2gnrlfile.hpp b/components/bsa/ba2gnrlfile.hpp index 8c0333d8d1..6a212c5e81 100644 --- a/components/bsa/ba2gnrlfile.hpp +++ b/components/bsa/ba2gnrlfile.hpp @@ -3,9 +3,10 @@ #include #include +#include +#include #include -#include namespace Bsa { From 19fbba080da522ec4fb3373911088d289d4a4d62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Mocquillon?= Date: Mon, 6 Mar 2023 21:26:51 +0100 Subject: [PATCH 11/11] Add a check if the requested file is not found --- components/bsa/ba2dx10file.cpp | 16 +++++++++------- components/bsa/ba2dx10file.hpp | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/components/bsa/ba2dx10file.cpp b/components/bsa/ba2dx10file.cpp index 4ec8359b85..af49c277ad 100644 --- a/components/bsa/ba2dx10file.cpp +++ b/components/bsa/ba2dx10file.cpp @@ -141,7 +141,7 @@ namespace Bsa mIsLoaded = true; } - BA2DX10File::FileRecord BA2DX10File::getFileRecord(const std::string& str) const + std::optional BA2DX10File::getFileRecord(const std::string& str) const { for (const auto c : str) { @@ -169,13 +169,13 @@ namespace Bsa uint32_t folderHash = generateHash(folder); auto it = mFolders.find(folderHash); if (it == mFolders.end()) - return FileRecord(); // folder not found, return default which has offset of sInvalidOffset + return std::nullopt; // folder not found uint32_t fileHash = generateHash(fileName); uint32_t extHash = *reinterpret_cast(ext.data() + 1); auto iter = it->second.find({ fileHash, extHash }); if (iter == it->second.end()) - return FileRecord(); // file not found, return default which has offset of sInvalidOffset + return std::nullopt; // file not found return iter->second; } @@ -221,8 +221,9 @@ namespace Bsa Files::IStreamPtr BA2DX10File::getFile(const FileStruct* file) { - FileRecord fileRec = getFileRecord(file->name()); - return getFile(fileRec); + if (auto fileRec = getFileRecord(file->name()); fileRec) + return getFile(*fileRec); + fail("File not found: " + std::string(file->name())); } void BA2DX10File::addFile(const std::string& filename, std::istream& file) @@ -233,8 +234,9 @@ namespace Bsa Files::IStreamPtr BA2DX10File::getFile(const char* file) { - FileRecord fileRec = getFileRecord(file); - return getFile(fileRec); + if (auto fileRec = getFileRecord(file); fileRec) + return getFile(*fileRec); + fail("File not found: " + std::string(file)); } constexpr const uint32_t DDSD_CAPS = 0x00000001; diff --git a/components/bsa/ba2dx10file.hpp b/components/bsa/ba2dx10file.hpp index 398e555d30..59db10745b 100644 --- a/components/bsa/ba2dx10file.hpp +++ b/components/bsa/ba2dx10file.hpp @@ -41,7 +41,7 @@ namespace Bsa std::list> mFileNames; - FileRecord getFileRecord(const std::string& str) const; + std::optional getFileRecord(const std::string& str) const; Files::IStreamPtr getFile(const FileRecord& fileRecord);