diff --git a/apps/essimporter/CMakeLists.txt b/apps/essimporter/CMakeLists.txt index 217d3e7b50..9f562017a0 100644 --- a/apps/essimporter/CMakeLists.txt +++ b/apps/essimporter/CMakeLists.txt @@ -36,6 +36,7 @@ openmw_add_executable(openmw-essimporter target_link_libraries(openmw-essimporter Boost::program_options components + openmw-lib ) if (BUILD_WITH_CODE_COVERAGE) diff --git a/apps/essimporter/converter.hpp b/apps/essimporter/converter.hpp index 520dd27f2a..953022ccb2 100644 --- a/apps/essimporter/converter.hpp +++ b/apps/essimporter/converter.hpp @@ -1,18 +1,21 @@ #ifndef OPENMW_ESSIMPORT_CONVERTER_H #define OPENMW_ESSIMPORT_CONVERTER_H +#include #include +#include #include #include -#include -#include - +#include #include #include #include +#include +#include #include +#include #include #include #include @@ -23,7 +26,6 @@ #include #include #include - #include #include "importcntc.hpp" @@ -45,11 +47,11 @@ #include "convertnpcc.hpp" #include "convertplayer.hpp" #include "convertscpt.hpp" -#include + +#include "apps/openmw/mwworld/esmstore.hpp" namespace ESSImport { - class Converter { public: @@ -59,6 +61,7 @@ namespace ESSImport virtual ~Converter() = default; void setContext(Context& context) { mContext = &context; } + void setDataStore(MWWorld::ESMStore& dataStore) { mDataStore = &dataStore; } /// @note The load method of ESM records accept the deleted flag as a parameter. /// I don't know can the DELE sub-record appear in saved games, so the deleted flag will be ignored. @@ -70,6 +73,12 @@ namespace ESSImport protected: Context* mContext; + MWWorld::ESMStore* mDataStore; + + std::string getGMST(std::string_view id) const + { + return mDataStore->get().find(id)->mValue.getString(); + } }; /// Default converter: simply reads the record and writes it unmodified to the output @@ -459,8 +468,8 @@ namespace ESSImport std::map> mStolenItems; }; - /// Seen responses for a dialogue topic? - /// Each DIAL record is followed by a number of INFO records, I believe, just like in ESMs + /// Each DIAL record is followed by a number of INFO records, just like in ESMs. + /// INFO records store the IDs of seen journal, quest, and topic entries. /// Dialogue conversion problems: /// - Journal is stored in one continuous HTML markup rather than each entry separately with associated info ID. /// - Seen dialogue responses only store the INFO id, rather than the fulltext. @@ -472,7 +481,117 @@ namespace ESSImport { INFO info; info.load(esm); + info.mTopic = mContext->mCurrentDialogueRefId; + mInfos.push_back(info); } + void write(ESM::ESMWriter& esm) override + { + for (const auto& info : mInfos) + { + ESM::JournalEntry entry; + entry.mTopic = info.mTopic; + entry.mInfo = info.mInfo; + entry.mActorName = info.mActorRefId; + + // Attempt to resolve actor name with data store + if (!entry.mActorName.empty()) + { + if (const ESM::NPC* npc = mDataStore->get().search(ESM::StringRefId(entry.mActorName))) + { + if (!npc->mName.empty()) + entry.mActorName = npc->mName; + } + else if (const ESM::Creature* creature + = mDataStore->get().search(ESM::StringRefId(entry.mActorName))) + { + if (!creature->mName.empty()) + entry.mActorName = creature->mName; + } + else + throw std::runtime_error("Unknown imported actor ID " + entry.mActorName + " for topic " + + entry.mTopic.toDebugString()); + } + + if (const ESM::Dialogue* dialogue = mDataStore->get().search(entry.mTopic)) + { + // Attempt to resolve response text with data store + for (const auto& infoEntry : dialogue->mInfo) + { + if (infoEntry.mId == entry.mInfo) + { + // We can't use the script interpreter because the environment is null + entry.mText = std::regex_replace( + infoEntry.mResponse, std::regex("%PCName"), mContext->mPlayerBase.mName); + entry.mText = std::regex_replace( + entry.mText, std::regex("%PCRace"), mContext->mPlayerBase.mRace.toString()); + entry.mText = std::regex_replace(entry.mText, std::regex("%Name"), entry.mActorName); + break; + } + } + + // Attempt to resolve record type with data store + switch (dialogue->mType) + { + case ESM::Dialogue::Journal: + entry.mType = ESM::JournalEntry::Type_Journal; + + if (auto it = mContext->mJournalEntries.find(entry.mText); + it != mContext->mJournalEntries.end()) + { + entry.mDayOfMonth = it->second.mDayOfMonth; + entry.mMonth = it->second.mMonth; + entry.mDay = it->second.mDay; + } + else + { + // We don't have a date, so default to 16 Last Seed (Day 1) + entry.mDayOfMonth = 16; + entry.mMonth = 7; + entry.mDay = 1; + } + break; + + case ESM::Dialogue::Topic: + entry.mType = ESM::JournalEntry::Type_Topic; + break; + + default: + throw std::runtime_error( + "Unrecognized dialogue record type for topic " + entry.mTopic.toDebugString()); + } + } + else + entry.mType = ESM::JournalEntry::Type_Topic; + + mOrderedEntries.emplace_back(entry); + } + std::stable_sort(mOrderedEntries.begin(), mOrderedEntries.end(), + [](const ESM::JournalEntry& a, const ESM::JournalEntry& b) { return a.mDay < b.mDay; }); + + for (const auto& entry : mOrderedEntries) + { + esm.startRecord(ESM::REC_JOUR); + entry.save(esm); + esm.endRecord(ESM::REC_JOUR); + + // Journal-type entries need a matching quest-type entry in OpenMW + if (entry.mType == ESM::JournalEntry::Type_Journal) + { + ESM::JournalEntry questEntry; + questEntry.mTopic = entry.mTopic; + questEntry.mInfo = entry.mInfo; + questEntry.mType = ESM::JournalEntry::Type_Quest; + + esm.startRecord(ESM::REC_JOUR); + questEntry.save(esm); + esm.endRecord(ESM::REC_JOUR); + } + } + } + + private: + std::vector mInfos; + std::vector mOrderedEntries; }; class ConvertDIAL : public Converter @@ -480,28 +599,43 @@ namespace ESSImport public: void read(ESM::ESMReader& esm) override { - std::string id = esm.getHNString("NAME"); + ESM::RefId refId = esm.getHNRefId("NAME"); + mContext->mCurrentDialogueRefId = refId; // Stored for subsequent INFO records DIAL dial; dial.load(esm); if (dial.mIndex > 0) - mDials[id] = dial; + mQuestStates[refId] = dial; } void write(ESM::ESMWriter& esm) override { - for (auto it = mDials.begin(); it != mDials.end(); ++it) + for (const auto& [refId, questState] : mQuestStates) { + int finished = 0; + if (const ESM::Dialogue* dialogue = mDataStore->get().search(refId)) + { + for (const auto& info : dialogue->mInfo) + { + if (info.mData.mJournalIndex == questState.mIndex) + { + // If this journal index is finished, the imported quest state should be also + finished = info.mQuestStatus == ESM::DialInfo::QuestStatus::QS_Finished ? 1 : 0; + break; + } + } + } + esm.startRecord(ESM::REC_QUES); ESM::QuestState state; - state.mFinished = 0; - state.mState = it->second.mIndex; - state.mTopic = ESM::RefId::stringRefId(it->first); + state.mFinished = finished; + state.mState = questState.mIndex; + state.mTopic = refId; state.save(esm); esm.endRecord(ESM::REC_QUES); } } private: - std::map mDials; + std::map mQuestStates; }; class ConvertQUES : public Converter @@ -509,8 +643,8 @@ namespace ESSImport public: void read(ESM::ESMReader& esm) override { - std::string id = esm.getHNString("NAME"); QUES quest; + quest.mName = esm.getHNString("NAME"); quest.load(esm); } }; @@ -522,6 +656,63 @@ namespace ESSImport { JOUR journal; journal.load(esm); + + // Strip formatting tokens and tags + std::string cleanedText + = std::regex_replace(journal.mText, std::regex(R"(@|#||<\/FONT>
|

)"), ""); + + std::istringstream stream(cleanedText); + std::string line, currentEntryText; + int currentHeaderDay = 0, currentHeaderDayOfMonth = 0, currentHeaderMonth = 0, lineNum = 0; + + // Load localized month names from the data store + std::map monthMap = { { getGMST("sMonthMorningstar"), 0 }, + { getGMST("sMonthSunsdawn"), 1 }, { getGMST("sMonthFirstseed"), 2 }, { getGMST("sMonthRainshand"), 3 }, + { getGMST("sMonthSecondseed"), 4 }, { getGMST("sMonthMidyear"), 5 }, { getGMST("sMonthSunsheight"), 6 }, + { getGMST("sMonthLastseed"), 7 }, { getGMST("sMonthHeartfire"), 8 }, { getGMST("sMonthFrostfall"), 9 }, + { getGMST("sMonthSunsdusk"), 10 }, { getGMST("sMonthEveningstar"), 11 } }; + + std::string sDay = getGMST("sDay"); + std::regex headerPattern(R"((\d{1,2})\s+([A-Za-z']+\s?[A-Za-z]*)\s+\()" + sDay + R"(\s+(\d+)\))"); + + while (std::getline(stream, line)) + { + ++lineNum; + + // All journal entries begin with a date header line + std::smatch matches; + if (std::regex_match(line, matches, headerPattern)) + { + if (lineNum > 1) + { + // Save the current entry when we reach the next header + ESM::JournalEntry entry; + entry.mDayOfMonth = currentHeaderDayOfMonth; + entry.mMonth = currentHeaderMonth; + entry.mDay = currentHeaderDay; + mContext->mJournalEntries[currentEntryText] = entry; + } + currentHeaderDayOfMonth = std::stoi(matches[1].str()); + currentHeaderMonth = monthMap[matches[2].str()]; + currentHeaderDay = std::stoi(matches[3].str()); + currentEntryText.clear(); + } + // Multi-line entry text follows the date header line + else + { + // Build the entry text between the headers + if (!currentEntryText.empty()) + currentEntryText += "\n"; + currentEntryText += line; + } + } + + // Save the final entry + ESM::JournalEntry lastEntry; + lastEntry.mDayOfMonth = currentHeaderDayOfMonth; + lastEntry.mMonth = currentHeaderMonth; + lastEntry.mDay = currentHeaderDay; + mContext->mJournalEntries[currentEntryText] = lastEntry; } }; diff --git a/apps/essimporter/importer.cpp b/apps/essimporter/importer.cpp index cf04fee163..3e5523843f 100644 --- a/apps/essimporter/importer.cpp +++ b/apps/essimporter/importer.cpp @@ -8,13 +8,9 @@ #include #include +#include #include #include - -#include -#include - -#include #include #include #include @@ -22,11 +18,14 @@ #include #include #include - +#include +#include +#include #include - #include +#include "apps/openmw/mwworld/esmstore.hpp" + #include "importercontext.hpp" #include "converter.hpp" @@ -87,11 +86,12 @@ namespace namespace ESSImport { - Importer::Importer( - const std::filesystem::path& essfile, const std::filesystem::path& outfile, const std::string& encoding) + Importer::Importer(const std::filesystem::path& essfile, const std::filesystem::path& outfile, + const std::string& encoding, const std::filesystem::path& datafilesPath) : mEssFile(essfile) , mOutFile(outfile) , mEncoding(encoding) + , mDataFilesPath(datafilesPath) { } @@ -259,12 +259,35 @@ namespace ESSImport { ToUTF8::Utf8Encoder encoder(ToUTF8::calculateEncoding(mEncoding)); ESM::ESMReader esm; - esm.open(mEssFile); esm.setEncoder(&encoder); - - Context context; + esm.open(mEssFile); const ESM::Header& header = esm.getHeader(); + + // Build the data store + MWWorld::ESMStore dataStore; + std::vector contentFiles; + for (const auto& master : header.mMaster) + { + contentFiles.push_back(master.name); + } + static Loading::Listener dummyListener; + + int index = 0; + ESM::Dialogue* dialogue = nullptr; + for (const auto& mContentFile : contentFiles) + { + ESM::ESMReader lEsm; + lEsm.setEncoder(&encoder); + lEsm.setIndex(index); + lEsm.open(mDataFilesPath.string() + "/" + mContentFile); + dataStore.load(lEsm, &dummyListener, dialogue); + + ++index; + } + dataStore.setUp(); + + Context context; context.mPlayerCellName = header.mGameData.mCurrentCell.toString(); const unsigned int recREFR = ESM::fourCC("REFR"); @@ -320,6 +343,7 @@ namespace ESSImport for (const auto& converter : converters) { converter.second->setContext(context); + converter.second->setDataStore(dataStore); } while (esm.hasMoreRecs()) diff --git a/apps/essimporter/importer.hpp b/apps/essimporter/importer.hpp index fba1992808..5345e83a93 100644 --- a/apps/essimporter/importer.hpp +++ b/apps/essimporter/importer.hpp @@ -9,8 +9,8 @@ namespace ESSImport class Importer { public: - Importer( - const std::filesystem::path& essfile, const std::filesystem::path& outfile, const std::string& encoding); + Importer(const std::filesystem::path& essfile, const std::filesystem::path& outfile, + const std::string& encoding, const std::filesystem::path& datafilesPath); void run(); @@ -20,6 +20,7 @@ namespace ESSImport std::filesystem::path mEssFile; std::filesystem::path mOutFile; std::string mEncoding; + std::filesystem::path mDataFilesPath; }; } diff --git a/apps/essimporter/importercontext.hpp b/apps/essimporter/importercontext.hpp index 03ea9d0943..510aeb23d0 100644 --- a/apps/essimporter/importercontext.hpp +++ b/apps/essimporter/importercontext.hpp @@ -3,9 +3,11 @@ #include +#include #include #include #include +#include #include #include #include @@ -28,7 +30,9 @@ namespace ESSImport ESM::NPC mPlayerBase; std::string mCustomPlayerClassName; + ESM::RefId mCurrentDialogueRefId; ESM::DialogueState mDialogueState; + std::map mJournalEntries; ESM::ControlsState mControlsState; diff --git a/apps/essimporter/importinfo.hpp b/apps/essimporter/importinfo.hpp index e0d1de4e92..c1f020c7fb 100644 --- a/apps/essimporter/importinfo.hpp +++ b/apps/essimporter/importinfo.hpp @@ -15,7 +15,10 @@ namespace ESSImport struct INFO { + ESM::RefId mTopic; ESM::RefId mInfo; + + // Treated as a string by ESM::JournalEntry std::string mActorRefId; void load(ESM::ESMReader& esm); diff --git a/apps/essimporter/main.cpp b/apps/essimporter/main.cpp index 50e11e2c5a..e9809de039 100644 --- a/apps/essimporter/main.cpp +++ b/apps/essimporter/main.cpp @@ -20,6 +20,7 @@ Allowed options)"); addOption("help,h", "produce help message"); addOption("mwsave,m", bpo::value(), "morrowind .ess save file"); addOption("output,o", bpo::value(), "output file (.omwsave)"); + addOption("dataFilePath,d", bpo::value(), "path to Morrowind data files directory"); addOption("compare,c", "compare two .ess files"); addOption("encoding", boost::program_options::value()->default_value("win1252"), "encoding of the save file"); @@ -46,8 +47,9 @@ Allowed options)"); const auto& essFile = variables["mwsave"].as(); const auto& outputFile = variables["output"].as(); std::string encoding = variables["encoding"].as(); + const auto& dataFilePath = variables["dataFilePath"].as(); - ESSImport::Importer importer(essFile, outputFile, encoding); + ESSImport::Importer importer(essFile, outputFile, encoding, dataFilePath); if (variables.count("compare")) importer.compare();