Merge branch 'ImporterImprovements' into 'master'

Draft: Import ESS journal, topic, and quest entries

Closes #2314

See merge request OpenMW/openmw!4717

apps/essimporter/CMakeLists.txt

@@ -36,6 +36,7 @@ openmw_add_executable(openmw-essimporter
 target_link_libraries(openmw-essimporter
     Boost::program_options
     components
+    openmw-lib
 )
 
 if (BUILD_WITH_CODE_COVERAGE)

apps/essimporter/converter.hpp

@@ -1,18 +1,21 @@
 #ifndef OPENMW_ESSIMPORT_CONVERTER_H
 #define OPENMW_ESSIMPORT_CONVERTER_H
 
+#include <algorithm>
 #include <limits>
+#include <regex>
 
 #include <osg/Image>
 #include <osg/ref_ptr>
 
-#include <components/esm3/esmreader.hpp>
+#include <components/esm/refid.hpp>
-#include <components/esm3/esmwriter.hpp>
-
 #include <components/esm3/cellstate.hpp>
 #include <components/esm3/custommarkerstate.hpp>
 #include <components/esm3/dialoguestate.hpp>
+#include <components/esm3/esmreader.hpp>
+#include <components/esm3/esmwriter.hpp>
 #include <components/esm3/globalscript.hpp>
+#include <components/esm3/journalentry.hpp>
 #include <components/esm3/loadbook.hpp>
 #include <components/esm3/loadcell.hpp>
 #include <components/esm3/loadclas.hpp>
@@ -23,7 +26,6 @@
 #include <components/esm3/queststate.hpp>
 #include <components/esm3/stolenitems.hpp>
 #include <components/esm3/weatherstate.hpp>
-
 #include <components/misc/strings/algorithm.hpp>
 
 #include "importcntc.hpp"
@@ -45,11 +47,11 @@
 #include "convertnpcc.hpp"
 #include "convertplayer.hpp"
 #include "convertscpt.hpp"
-#include <components/esm/refid.hpp>
+
+#include "apps/openmw/mwworld/esmstore.hpp"
 
 namespace ESSImport
 {
-
     class Converter
     {
     public:
@@ -59,6 +61,7 @@ namespace ESSImport
         virtual ~Converter() = default;
 
         void setContext(Context& context) { mContext = &context; }
+        void setDataStore(MWWorld::ESMStore& dataStore) { mDataStore = &dataStore; }
 
         /// @note The load method of ESM records accept the deleted flag as a parameter.
         /// I don't know can the DELE sub-record appear in saved games, so the deleted flag will be ignored.
@@ -70,6 +73,12 @@ namespace ESSImport
 
     protected:
         Context* mContext;
+        MWWorld::ESMStore* mDataStore;
+
+        std::string getGMST(std::string_view id) const
+        {
+            return mDataStore->get<ESM::GameSetting>().find(id)->mValue.getString();
+        }
     };
 
     /// Default converter: simply reads the record and writes it unmodified to the output
@@ -459,8 +468,8 @@ namespace ESSImport
         std::map<std::string, std::set<Owner>> mStolenItems;
     };
 
-    /// Seen responses for a dialogue topic?
+    /// Each DIAL record is followed by a number of INFO records, just like in ESMs.
-    /// Each DIAL record is followed by a number of INFO records, I believe, just like in ESMs
+    /// INFO records store the IDs of seen journal, quest, and topic entries.
     /// Dialogue conversion problems:
     /// - Journal is stored in one continuous HTML markup rather than each entry separately with associated info ID.
     /// - Seen dialogue responses only store the INFO id, rather than the fulltext.
@@ -472,7 +481,117 @@ namespace ESSImport
         {
             INFO info;
             info.load(esm);
+            info.mTopic = mContext->mCurrentDialogueRefId;
+            mInfos.push_back(info);
         }
+        void write(ESM::ESMWriter& esm) override
+        {
+            for (const auto& info : mInfos)
+            {
+                ESM::JournalEntry entry;
+                entry.mTopic = info.mTopic;
+                entry.mInfo = info.mInfo;
+                entry.mActorName = info.mActorRefId;
+
+                // Attempt to resolve actor name with data store
+                if (!entry.mActorName.empty())
+                {
+                    if (const ESM::NPC* npc = mDataStore->get<ESM::NPC>().search(ESM::StringRefId(entry.mActorName)))
+                    {
+                        if (!npc->mName.empty())
+                            entry.mActorName = npc->mName;
+                    }
+                    else if (const ESM::Creature* creature
+                        = mDataStore->get<ESM::Creature>().search(ESM::StringRefId(entry.mActorName)))
+                    {
+                        if (!creature->mName.empty())
+                            entry.mActorName = creature->mName;
+                    }
+                    else
+                        throw std::runtime_error("Unknown imported actor ID " + entry.mActorName + " for topic "
+                            + entry.mTopic.toDebugString());
+                }
+
+                if (const ESM::Dialogue* dialogue = mDataStore->get<ESM::Dialogue>().search(entry.mTopic))
+                {
+                    // Attempt to resolve response text with data store
+                    for (const auto& infoEntry : dialogue->mInfo)
+                    {
+                        if (infoEntry.mId == entry.mInfo)
+                        {
+                            // We can't use the script interpreter because the environment is null
+                            entry.mText = std::regex_replace(
+                                infoEntry.mResponse, std::regex("%PCName"), mContext->mPlayerBase.mName);
+                            entry.mText = std::regex_replace(
+                                entry.mText, std::regex("%PCRace"), mContext->mPlayerBase.mRace.toString());
+                            entry.mText = std::regex_replace(entry.mText, std::regex("%Name"), entry.mActorName);
+                            break;
+                        }
+                    }
+
+                    // Attempt to resolve record type with data store
+                    switch (dialogue->mType)
+                    {
+                        case ESM::Dialogue::Journal:
+                            entry.mType = ESM::JournalEntry::Type_Journal;
+
+                            if (auto it = mContext->mJournalEntries.find(entry.mText);
+                                it != mContext->mJournalEntries.end())
+                            {
+                                entry.mDayOfMonth = it->second.mDayOfMonth;
+                                entry.mMonth = it->second.mMonth;
+                                entry.mDay = it->second.mDay;
+                            }
+                            else
+                            {
+                                // We don't have a date, so default to 16 Last Seed (Day 1)
+                                entry.mDayOfMonth = 16;
+                                entry.mMonth = 7;
+                                entry.mDay = 1;
+                            }
+                            break;
+
+                        case ESM::Dialogue::Topic:
+                            entry.mType = ESM::JournalEntry::Type_Topic;
+                            break;
+
+                        default:
+                            throw std::runtime_error(
+                                "Unrecognized dialogue record type for topic " + entry.mTopic.toDebugString());
+                    }
+                }
+                else
+                    entry.mType = ESM::JournalEntry::Type_Topic;
+
+                mOrderedEntries.emplace_back(entry);
+            }
+            std::stable_sort(mOrderedEntries.begin(), mOrderedEntries.end(),
+                [](const ESM::JournalEntry& a, const ESM::JournalEntry& b) { return a.mDay < b.mDay; });
+
+            for (const auto& entry : mOrderedEntries)
+            {
+                esm.startRecord(ESM::REC_JOUR);
+                entry.save(esm);
+                esm.endRecord(ESM::REC_JOUR);
+
+                // Journal-type entries need a matching quest-type entry in OpenMW
+                if (entry.mType == ESM::JournalEntry::Type_Journal)
+                {
+                    ESM::JournalEntry questEntry;
+                    questEntry.mTopic = entry.mTopic;
+                    questEntry.mInfo = entry.mInfo;
+                    questEntry.mType = ESM::JournalEntry::Type_Quest;
+
+                    esm.startRecord(ESM::REC_JOUR);
+                    questEntry.save(esm);
+                    esm.endRecord(ESM::REC_JOUR);
+                }
+            }
+        }
+
+    private:
+        std::vector<INFO> mInfos;
+        std::vector<ESM::JournalEntry> mOrderedEntries;
     };
 
     class ConvertDIAL : public Converter
@@ -480,28 +599,43 @@ namespace ESSImport
     public:
         void read(ESM::ESMReader& esm) override
         {
-            std::string id = esm.getHNString("NAME");
+            ESM::RefId refId = esm.getHNRefId("NAME");
+            mContext->mCurrentDialogueRefId = refId; // Stored for subsequent INFO records
             DIAL dial;
             dial.load(esm);
             if (dial.mIndex > 0)
-                mDials[id] = dial;
+                mQuestStates[refId] = dial;
         }
         void write(ESM::ESMWriter& esm) override
         {
-            for (auto it = mDials.begin(); it != mDials.end(); ++it)
+            for (const auto& [refId, questState] : mQuestStates)
             {
+                int finished = 0;
+                if (const ESM::Dialogue* dialogue = mDataStore->get<ESM::Dialogue>().search(refId))
+                {
+                    for (const auto& info : dialogue->mInfo)
+                    {
+                        if (info.mData.mJournalIndex == questState.mIndex)
+                        {
+                            // If this journal index is finished, the imported quest state should be also
+                            finished = info.mQuestStatus == ESM::DialInfo::QuestStatus::QS_Finished ? 1 : 0;
+                            break;
+                        }
+                    }
+                }
+
                 esm.startRecord(ESM::REC_QUES);
                 ESM::QuestState state;
-                state.mFinished = 0;
+                state.mFinished = finished;
-                state.mState = it->second.mIndex;
+                state.mState = questState.mIndex;
-                state.mTopic = ESM::RefId::stringRefId(it->first);
+                state.mTopic = refId;
                 state.save(esm);
                 esm.endRecord(ESM::REC_QUES);
             }
         }
 
     private:
-        std::map<std::string, DIAL> mDials;
+        std::map<ESM::RefId, DIAL> mQuestStates;
     };
 
     class ConvertQUES : public Converter
@@ -509,8 +643,8 @@ namespace ESSImport
     public:
         void read(ESM::ESMReader& esm) override
         {
-            std::string id = esm.getHNString("NAME");
             QUES quest;
+            quest.mName = esm.getHNString("NAME");
             quest.load(esm);
         }
     };
@@ -522,6 +656,63 @@ namespace ESSImport
         {
             JOUR journal;
             journal.load(esm);
+
+            // Strip formatting tokens and tags
+            std::string cleanedText
+                = std::regex_replace(journal.mText, std::regex(R"(@|#|<FONT COLOR="9F0000">|<\/FONT><BR>|<P>)"), "");
+
+            std::istringstream stream(cleanedText);
+            std::string line, currentEntryText;
+            int currentHeaderDay = 0, currentHeaderDayOfMonth = 0, currentHeaderMonth = 0, lineNum = 0;
+
+            // Load localized month names from the data store
+            std::map<std::string, int> monthMap = { { getGMST("sMonthMorningstar"), 0 },
+                { getGMST("sMonthSunsdawn"), 1 }, { getGMST("sMonthFirstseed"), 2 }, { getGMST("sMonthRainshand"), 3 },
+                { getGMST("sMonthSecondseed"), 4 }, { getGMST("sMonthMidyear"), 5 }, { getGMST("sMonthSunsheight"), 6 },
+                { getGMST("sMonthLastseed"), 7 }, { getGMST("sMonthHeartfire"), 8 }, { getGMST("sMonthFrostfall"), 9 },
+                { getGMST("sMonthSunsdusk"), 10 }, { getGMST("sMonthEveningstar"), 11 } };
+
+            std::string sDay = getGMST("sDay");
+            std::regex headerPattern(R"((\d{1,2})\s+([A-Za-z']+\s?[A-Za-z]*)\s+\()" + sDay + R"(\s+(\d+)\))");
+
+            while (std::getline(stream, line))
+            {
+                ++lineNum;
+
+                // All journal entries begin with a date header line
+                std::smatch matches;
+                if (std::regex_match(line, matches, headerPattern))
+                {
+                    if (lineNum > 1)
+                    {
+                        // Save the current entry when we reach the next header
+                        ESM::JournalEntry entry;
+                        entry.mDayOfMonth = currentHeaderDayOfMonth;
+                        entry.mMonth = currentHeaderMonth;
+                        entry.mDay = currentHeaderDay;
+                        mContext->mJournalEntries[currentEntryText] = entry;
+                    }
+                    currentHeaderDayOfMonth = std::stoi(matches[1].str());
+                    currentHeaderMonth = monthMap[matches[2].str()];
+                    currentHeaderDay = std::stoi(matches[3].str());
+                    currentEntryText.clear();
+                }
+                // Multi-line entry text follows the date header line
+                else
+                {
+                    // Build the entry text between the headers
+                    if (!currentEntryText.empty())
+                        currentEntryText += "\n";
+                    currentEntryText += line;
+                }
+            }
+
+            // Save the final entry
+            ESM::JournalEntry lastEntry;
+            lastEntry.mDayOfMonth = currentHeaderDayOfMonth;
+            lastEntry.mMonth = currentHeaderMonth;
+            lastEntry.mDay = currentHeaderDay;
+            mContext->mJournalEntries[currentEntryText] = lastEntry;
         }
     };
 

apps/essimporter/importer.cpp

@@ -8,13 +8,9 @@
 #include <osgDB/ReadFile>
 
 #include <components/esm/defs.hpp>
+#include <components/esm3/cellid.hpp>
 #include <components/esm3/esmreader.hpp>
 #include <components/esm3/esmwriter.hpp>
-
-#include <components/esm3/player.hpp>
-#include <components/esm3/savedgame.hpp>
-
-#include <components/esm3/cellid.hpp>
 #include <components/esm3/loadalch.hpp>
 #include <components/esm3/loadarmo.hpp>
 #include <components/esm3/loadclot.hpp>
@@ -22,11 +18,14 @@
 #include <components/esm3/loadlevlist.hpp>
 #include <components/esm3/loadspel.hpp>
 #include <components/esm3/loadweap.hpp>
-
+#include <components/esm3/player.hpp>
+#include <components/esm3/savedgame.hpp>
+#include <components/loadinglistener/loadinglistener.hpp>
 #include <components/misc/constants.hpp>
-
 #include <components/toutf8/toutf8.hpp>
 
+#include "apps/openmw/mwworld/esmstore.hpp"
+
 #include "importercontext.hpp"
 
 #include "converter.hpp"
@@ -87,11 +86,12 @@ namespace
 namespace ESSImport
 {
 
-    Importer::Importer(
+    Importer::Importer(const std::filesystem::path& essfile, const std::filesystem::path& outfile,
-        const std::filesystem::path& essfile, const std::filesystem::path& outfile, const std::string& encoding)
+        const std::string& encoding, const std::filesystem::path& datafilesPath)
         : mEssFile(essfile)
         , mOutFile(outfile)
         , mEncoding(encoding)
+        , mDataFilesPath(datafilesPath)
     {
     }
 
@@ -259,12 +259,35 @@ namespace ESSImport
     {
         ToUTF8::Utf8Encoder encoder(ToUTF8::calculateEncoding(mEncoding));
         ESM::ESMReader esm;
-        esm.open(mEssFile);
         esm.setEncoder(&encoder);
-
+        esm.open(mEssFile);
-        Context context;
 
         const ESM::Header& header = esm.getHeader();
+
+        // Build the data store
+        MWWorld::ESMStore dataStore;
+        std::vector<std::string> contentFiles;
+        for (const auto& master : header.mMaster)
+        {
+            contentFiles.push_back(master.name);
+        }
+        static Loading::Listener dummyListener;
+
+        int index = 0;
+        ESM::Dialogue* dialogue = nullptr;
+        for (const auto& mContentFile : contentFiles)
+        {
+            ESM::ESMReader lEsm;
+            lEsm.setEncoder(&encoder);
+            lEsm.setIndex(index);
+            lEsm.open(mDataFilesPath.string() + "/" + mContentFile);
+            dataStore.load(lEsm, &dummyListener, dialogue);
+
+            ++index;
+        }
+        dataStore.setUp();
+
+        Context context;
         context.mPlayerCellName = header.mGameData.mCurrentCell.toString();
 
         const unsigned int recREFR = ESM::fourCC("REFR");
@@ -320,6 +343,7 @@ namespace ESSImport
         for (const auto& converter : converters)
         {
             converter.second->setContext(context);
+            converter.second->setDataStore(dataStore);
         }
 
         while (esm.hasMoreRecs())

apps/essimporter/importer.hpp

@@ -9,8 +9,8 @@ namespace ESSImport
     class Importer
     {
     public:
-        Importer(
+        Importer(const std::filesystem::path& essfile, const std::filesystem::path& outfile,
-            const std::filesystem::path& essfile, const std::filesystem::path& outfile, const std::string& encoding);
+            const std::string& encoding, const std::filesystem::path& datafilesPath);
 
         void run();
 
@@ -20,6 +20,7 @@ namespace ESSImport
         std::filesystem::path mEssFile;
         std::filesystem::path mOutFile;
         std::string mEncoding;
+        std::filesystem::path mDataFilesPath;
     };
 
 }

apps/essimporter/importercontext.hpp

@@ -3,9 +3,11 @@
 
 #include <map>
 
+#include <components/esm/refid.hpp>
 #include <components/esm3/controlsstate.hpp>
 #include <components/esm3/dialoguestate.hpp>
 #include <components/esm3/globalmap.hpp>
+#include <components/esm3/journalentry.hpp>
 #include <components/esm3/loadcell.hpp>
 #include <components/esm3/loadcrea.hpp>
 #include <components/esm3/loadnpc.hpp>
@@ -28,7 +30,9 @@ namespace ESSImport
         ESM::NPC mPlayerBase;
         std::string mCustomPlayerClassName;
 
+        ESM::RefId mCurrentDialogueRefId;
         ESM::DialogueState mDialogueState;
+        std::map<std::string, ESM::JournalEntry> mJournalEntries;
 
         ESM::ControlsState mControlsState;
 

apps/essimporter/importinfo.hpp

@@ -15,7 +15,10 @@ namespace ESSImport
 
     struct INFO
     {
+        ESM::RefId mTopic;
         ESM::RefId mInfo;
+
+        // Treated as a string by ESM::JournalEntry
         std::string mActorRefId;
 
         void load(ESM::ESMReader& esm);

apps/essimporter/main.cpp

@@ -20,6 +20,7 @@ Allowed options)");
         addOption("help,h", "produce help message");
         addOption("mwsave,m", bpo::value<Files::MaybeQuotedPath>(), "morrowind .ess save file");
         addOption("output,o", bpo::value<Files::MaybeQuotedPath>(), "output file (.omwsave)");
+        addOption("dataFilePath,d", bpo::value<Files::MaybeQuotedPath>(), "path to Morrowind data files directory");
         addOption("compare,c", "compare two .ess files");
         addOption("encoding", boost::program_options::value<std::string>()->default_value("win1252"),
             "encoding of the save file");
@@ -46,8 +47,9 @@ Allowed options)");
         const auto& essFile = variables["mwsave"].as<Files::MaybeQuotedPath>();
         const auto& outputFile = variables["output"].as<Files::MaybeQuotedPath>();
         std::string encoding = variables["encoding"].as<std::string>();
+        const auto& dataFilePath = variables["dataFilePath"].as<Files::MaybeQuotedPath>();
 
-        ESSImport::Importer importer(essFile, outputFile, encoding);
+        ESSImport::Importer importer(essFile, outputFile, encoding, dataFilePath);
 
         if (variables.count("compare"))
             importer.compare();