From e6196c782db5e90a31b71d4a5b619e9c8ba04fef Mon Sep 17 00:00:00 2001
From: elsid <elsid.mail@gmail.com>
Date: Tue, 30 Jan 2024 23:59:58 +0100
Subject: [PATCH] Limit navmesh vertices coordinates values

Float values with more than 22 significant fraction bits may cause out of bounds
access in recastnavigation on triangles rasterization. Prevent passing such
values there.
---
 CHANGELOG.md                                  |  1 +
 .../detournavigator/navigator.cpp             |  2 +-
 components/detournavigator/makenavmesh.cpp    | 28 +++++++++++++++++--
 3 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 65c42bc552..5658fdc904 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -37,6 +37,7 @@
     Bug #6402: The sound of a thunderstorm does not stop playing after entering the premises
     Bug #6427: Enemy health bar disappears before damaging effect ends
     Bug #6550: Cloned body parts don't inherit texture effects
+    Bug #6574: Crash at far away from world origin coordinates
     Bug #6645: Enemy block sounds align with animation instead of blocked hits
     Bug #6657: Distant terrain tiles become black when using FWIW mod
     Bug #6661: Saved games that have no preview screenshot cause issues or crashes
diff --git a/apps/openmw_test_suite/detournavigator/navigator.cpp b/apps/openmw_test_suite/detournavigator/navigator.cpp
index aba8598f18..b61a88662b 100644
--- a/apps/openmw_test_suite/detournavigator/navigator.cpp
+++ b/apps/openmw_test_suite/detournavigator/navigator.cpp
@@ -871,7 +871,7 @@ namespace
 
     TEST_F(DetourNavigatorNavigatorTest, update_for_very_big_object_should_be_limited)
     {
-        const float size = static_cast<float>(2 * static_cast<std::int64_t>(std::numeric_limits<int>::max()) - 1);
+        const float size = static_cast<float>((1 << 22) - 1);
         CollisionShapeInstance bigBox(std::make_unique<btBoxShape>(btVector3(size, size, 1)));
         const ObjectTransform objectTransform{
             .mPosition = ESM::Position{ .pos = { 0, 0, 0 }, .rot{ 0, 0, 0 } },
diff --git a/components/detournavigator/makenavmesh.cpp b/components/detournavigator/makenavmesh.cpp
index 4bac85f420..d35ecf499d 100644
--- a/components/detournavigator/makenavmesh.cpp
+++ b/components/detournavigator/makenavmesh.cpp
@@ -186,16 +186,35 @@ namespace DetourNavigator
                 &context, solid, width, height, bmin.ptr(), bmax.ptr(), settings.mCellSize, settings.mCellHeight);
         }
 
+        bool isSupportedCoordinate(float value)
+        {
+            constexpr float maxVertexCoordinate = static_cast<float>(1 << 22);
+            return -maxVertexCoordinate < value && value < maxVertexCoordinate;
+        }
+
+        template <class Iterator>
+        bool isSupportedCoordinates(Iterator begin, Iterator end)
+        {
+            return std::all_of(begin, end, isSupportedCoordinate);
+        }
+
         [[nodiscard]] bool rasterizeTriangles(RecastContext& context, const Mesh& mesh, const RecastSettings& settings,
             const RecastParams& params, rcHeightfield& solid)
         {
             std::vector<unsigned char> areas(mesh.getAreaTypes().begin(), mesh.getAreaTypes().end());
             std::vector<float> vertices = mesh.getVertices();
 
-            for (std::size_t i = 0; i < vertices.size(); i += 3)
+            constexpr std::size_t verticesPerTriangle = 3;
+
+            for (std::size_t i = 0; i < vertices.size(); i += verticesPerTriangle)
             {
-                for (std::size_t j = 0; j < 3; ++j)
-                    vertices[i + j] = toNavMeshCoordinates(settings, vertices[i + j]);
+                for (std::size_t j = 0; j < verticesPerTriangle; ++j)
+                {
+                    const float coordinate = toNavMeshCoordinates(settings, vertices[i + j]);
+                    if (!isSupportedCoordinate(coordinate))
+                        return false;
+                    vertices[i + j] = coordinate;
+                }
                 std::swap(vertices[i + 1], vertices[i + 2]);
             }
 
@@ -217,6 +236,9 @@ namespace DetourNavigator
                 rectangle.mBounds.mMax.x(), rectangle.mHeight, rectangle.mBounds.mMin.y(), // vertex 3
             };
 
+            if (!isSupportedCoordinates(vertices.begin(), vertices.end()))
+                return false;
+
             const std::array indices{
                 0, 1, 2, // triangle 0
                 0, 2, 3, // triangle 1