Merge branch 'bsazoomies' into 'master'

Use zlib directly for BSA file decompression (#8569)

Closes #8569

See merge request OpenMW/openmw!4795

components/bsa/ba2dx10file.cpp

@@ -1,29 +1,12 @@
 #include "ba2dx10file.hpp"
 
+#include <algorithm>
 #include <cassert>
+#include <cstring>
 #include <filesystem>
 #include <fstream>
 
-#include <lz4frame.h>
+#include <zlib.h>
-
-#if defined(_MSC_VER)
-// why is this necessary? These are included with /external:I
-#pragma warning(push)
-#pragma warning(disable : 4706)
-#pragma warning(disable : 4702)
-#include <boost/iostreams/copy.hpp>
-#include <boost/iostreams/filter/zlib.hpp>
-#include <boost/iostreams/filtering_stream.hpp>
-#include <boost/iostreams/filtering_streambuf.hpp>
-#pragma warning(pop)
-#else
-#include <boost/iostreams/copy.hpp>
-#include <boost/iostreams/filter/zlib.hpp>
-#include <boost/iostreams/filtering_stream.hpp>
-#include <boost/iostreams/filtering_streambuf.hpp>
-#endif
-
-#include <boost/iostreams/device/array.hpp>
 
 #include <components/esm/fourcc.hpp>
 #include <components/files/constrainedfilestream.hpp>
@@ -644,11 +627,16 @@ namespace Bsa
         size_t headerSize = (header.ddspf.fourCC == ESM::fourCC("DX10") ? sizeof(DDSHeaderDX10) : sizeof(DDSHeader));
 
         size_t textureSize = sizeof(uint32_t) + headerSize; //"DDS " + header
+        uint32_t maxPackedChunkSize = 0;
         for (const auto& textureChunk : fileRecord.texturesChunks)
+        {
             textureSize += textureChunk.size;
+            maxPackedChunkSize = std::max(textureChunk.packedSize, maxPackedChunkSize);
+        }
 
         auto memoryStreamPtr = std::make_unique<MemoryInputStream>(textureSize);
         char* buff = memoryStreamPtr->getRawData();
+        std::vector<char> inputBuffer(maxPackedChunkSize);
 
         uint32_t dds = ESM::fourCC("DDS ");
         buff = (char*)std::memcpy(buff, &dds, sizeof(uint32_t)) + sizeof(uint32_t);
@@ -658,25 +646,22 @@ namespace Bsa
         // append chunks
         for (const auto& c : fileRecord.texturesChunks)
         {
+            const uint32_t inputSize = c.packedSize != 0 ? c.packedSize : c.size;
+            Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilepath, c.offset, inputSize);
             if (c.packedSize != 0)
             {
-                Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilepath, c.offset, c.packedSize);
+                streamPtr->read(inputBuffer.data(), c.packedSize);
-                std::istream* fileStream = streamPtr.get();
+                uLongf destSize = static_cast<uLongf>(c.size);
+                int ec = ::uncompress(reinterpret_cast<Bytef*>(memoryStreamPtr->getRawData() + offset), &destSize,
+                    reinterpret_cast<Bytef*>(inputBuffer.data()), static_cast<uLong>(c.packedSize));
 
-                boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
+                if (ec != Z_OK)
-                inputStreamBuf.push(boost::iostreams::zlib_decompressor());
+                    fail("zlib uncompress failed: " + std::string(::zError(ec)));
-                inputStreamBuf.push(*fileStream);
-
-                boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData() + offset, c.size);
-                boost::iostreams::copy(inputStreamBuf, sr);
             }
             // uncompressed chunk
             else
             {
-                Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilepath, c.offset, c.size);
+                streamPtr->read(memoryStreamPtr->getRawData() + offset, c.size);
-                std::istream* fileStream = streamPtr.get();
-
-                fileStream->read(memoryStreamPtr->getRawData() + offset, c.size);
             }
             offset += c.size;
         }

components/bsa/ba2gnrlfile.cpp

@@ -1,27 +1,11 @@
 #include "ba2gnrlfile.hpp"
 
+#include <algorithm>
 #include <cassert>
 #include <filesystem>
 #include <fstream>
 
-#include <lz4frame.h>
+#include <zlib.h>
-
-#if defined(_MSC_VER)
-// why is this necessary? These are included with /external:I
-#pragma warning(push)
-#pragma warning(disable : 4706)
-#pragma warning(disable : 4702)
-#include <boost/iostreams/copy.hpp>
-#include <boost/iostreams/filter/zlib.hpp>
-#include <boost/iostreams/filtering_streambuf.hpp>
-#pragma warning(pop)
-#else
-#include <boost/iostreams/copy.hpp>
-#include <boost/iostreams/filter/zlib.hpp>
-#include <boost/iostreams/filtering_streambuf.hpp>
-#endif
-
-#include <boost/iostreams/device/array.hpp>
 
 #include <components/esm/fourcc.hpp>
 #include <components/files/constrainedfilestream.hpp>
@@ -223,12 +207,14 @@ namespace Bsa
         auto memoryStreamPtr = std::make_unique<MemoryInputStream>(fileRecord.size);
         if (fileRecord.packedSize)
         {
-            boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
+            std::vector<char> buffer(inputSize);
-            inputStreamBuf.push(boost::iostreams::zlib_decompressor());
+            streamPtr->read(buffer.data(), inputSize);
-            inputStreamBuf.push(*streamPtr);
+            uLongf destSize = static_cast<uLongf>(fileRecord.size);
+            int ec = ::uncompress(reinterpret_cast<Bytef*>(memoryStreamPtr->getRawData()), &destSize,
+                reinterpret_cast<Bytef*>(buffer.data()), static_cast<uLong>(buffer.size()));
 
-            boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData(), fileRecord.size);
+            if (ec != Z_OK)
-            boost::iostreams::copy(inputStreamBuf, sr);
+                fail("zlib uncompress failed: " + std::string(::zError(ec)));
         }
         else
         {

components/bsa/compressedbsafile.cpp

@@ -24,27 +24,13 @@
  */
 #include "compressedbsafile.hpp"
 
+#include <algorithm>
 #include <cassert>
 #include <filesystem>
 #include <fstream>
 
 #include <lz4frame.h>
-
+#include <zlib.h>
-#if defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable : 4706)
-#pragma warning(disable : 4702)
-#include <boost/iostreams/copy.hpp>
-#include <boost/iostreams/filter/zlib.hpp>
-#include <boost/iostreams/filtering_streambuf.hpp>
-#pragma warning(pop)
-#else
-#include <boost/iostreams/copy.hpp>
-#include <boost/iostreams/filter/zlib.hpp>
-#include <boost/iostreams/filtering_streambuf.hpp>
-#endif
-
-#include <boost/iostreams/device/array.hpp>
 
 #include <components/files/constrainedfilestream.hpp>
 #include <components/files/conversion.hpp>
@@ -292,19 +278,26 @@ namespace Bsa
 
         if (compressed)
         {
+            std::vector<char> buffer(size);
+            streamPtr->read(buffer.data(), size);
+
             if (mHeader.mVersion != Version_SSE)
             {
-                boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
+                uLongf destSize = static_cast<uLongf>(resultSize);
-                inputStreamBuf.push(boost::iostreams::zlib_decompressor());
+                int ec = ::uncompress(reinterpret_cast<Bytef*>(memoryStreamPtr->getRawData()), &destSize,
-                inputStreamBuf.push(*streamPtr);
+                    reinterpret_cast<Bytef*>(buffer.data()), static_cast<uLong>(buffer.size()));
 
-                boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData(), resultSize);
+                if (ec != Z_OK)
-                boost::iostreams::copy(inputStreamBuf, sr);
+                {
+                    std::string message = "zlib uncompress failed for file ";
+                    message.append(fileRecord.mName.begin(), fileRecord.mName.end());
+                    message += ": ";
+                    message += ::zError(ec);
+                    fail(message);
+                }
             }
             else
             {
-                auto buffer = std::vector<char>(size);
-                streamPtr->read(buffer.data(), size);
                 LZ4F_decompressionContext_t context = nullptr;
                 LZ4F_createDecompressionContext(&context, LZ4F_VERSION);
                 LZ4F_decompressOptions_t options = {};