AnarchistGamers/openmw
1
0
Fork 0
mirror of https://gitlab.com/OpenMW/openmw.git synced 2025-08-03 15:27:13 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'ImporterImprovements' into 'master'

Browse Source 
Draft: Import ESS journal, topic, and quest entries

Closes #2314

See merge request OpenMW/openmw!4717
This commit is contained in:
Aussiemon 2025-08-02 02:41:08 -06:00
commit 8671687513
7 changed files with 257 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/essimporter/CMakeLists.txt
View File

@ -36,6 +36,7 @@ openmw_add_executable(openmw-essimporter
target_link_libraries(openmw-essimporter
Boost::program_options
components
openmw-lib
)
if (BUILD_WITH_CODE_COVERAGE)

223
apps/essimporter/converter.hpp
View File

@ -1,18 +1,21 @@
#ifndef OPENMW_ESSIMPORT_CONVERTER_H
#define OPENMW_ESSIMPORT_CONVERTER_H
#include <algorithm>
#include <limits>
#include <regex>
#include <osg/Image>
#include <osg/ref_ptr>
#include <components/esm3/esmreader.hpp>
#include <components/esm3/esmwriter.hpp>
#include <components/esm/refid.hpp>
#include <components/esm3/cellstate.hpp>
#include <components/esm3/custommarkerstate.hpp>
#include <components/esm3/dialoguestate.hpp>
#include <components/esm3/esmreader.hpp>
#include <components/esm3/esmwriter.hpp>
#include <components/esm3/globalscript.hpp>
#include <components/esm3/journalentry.hpp>
#include <components/esm3/loadbook.hpp>
#include <components/esm3/loadcell.hpp>
#include <components/esm3/loadclas.hpp>
@ -23,7 +26,6 @@
#include <components/esm3/queststate.hpp>
#include <components/esm3/stolenitems.hpp>
#include <components/esm3/weatherstate.hpp>
#include <components/misc/strings/algorithm.hpp>
#include "importcntc.hpp"
@ -45,11 +47,11 @@
#include "convertnpcc.hpp"
#include "convertplayer.hpp"
#include "convertscpt.hpp"
#include <components/esm/refid.hpp>
#include "apps/openmw/mwworld/esmstore.hpp"
namespace ESSImport
{
class Converter
{
public:
@ -59,6 +61,7 @@ namespace ESSImport
virtual ~Converter() = default;
void setContext(Context& context) { mContext = &context; }
void setDataStore(MWWorld::ESMStore& dataStore) { mDataStore = &dataStore; }
/// @note The load method of ESM records accept the deleted flag as a parameter.
/// I don't know can the DELE sub-record appear in saved games, so the deleted flag will be ignored.
@ -70,6 +73,12 @@ namespace ESSImport
protected:
Context* mContext;
MWWorld::ESMStore* mDataStore;
std::string getGMST(std::string_view id) const
{
return mDataStore->get<ESM::GameSetting>().find(id)->mValue.getString();
}
};
/// Default converter: simply reads the record and writes it unmodified to the output
@ -459,8 +468,8 @@ namespace ESSImport
std::map<std::string, std::set<Owner>> mStolenItems;
};
/// Seen responses for a dialogue topic?
/// Each DIAL record is followed by a number of INFO records, I believe, just like in ESMs
/// Each DIAL record is followed by a number of INFO records, just like in ESMs.
/// INFO records store the IDs of seen journal, quest, and topic entries.
/// Dialogue conversion problems:
/// - Journal is stored in one continuous HTML markup rather than each entry separately with associated info ID.
/// - Seen dialogue responses only store the INFO id, rather than the fulltext.
@ -472,7 +481,117 @@ namespace ESSImport
{
INFO info;
info.load(esm);
info.mTopic = mContext->mCurrentDialogueRefId;
mInfos.push_back(info);
}
void write(ESM::ESMWriter& esm) override
{
for (const auto& info : mInfos)
{
ESM::JournalEntry entry;
entry.mTopic = info.mTopic;
entry.mInfo = info.mInfo;
entry.mActorName = info.mActorRefId;
// Attempt to resolve actor name with data store
if (!entry.mActorName.empty())
{
if (const ESM::NPC* npc = mDataStore->get<ESM::NPC>().search(ESM::StringRefId(entry.mActorName)))
{
if (!npc->mName.empty())
entry.mActorName = npc->mName;
}
else if (const ESM::Creature* creature
= mDataStore->get<ESM::Creature>().search(ESM::StringRefId(entry.mActorName)))
{
if (!creature->mName.empty())
entry.mActorName = creature->mName;
}
else
throw std::runtime_error("Unknown imported actor ID " + entry.mActorName + " for topic "
+ entry.mTopic.toDebugString());
}
if (const ESM::Dialogue* dialogue = mDataStore->get<ESM::Dialogue>().search(entry.mTopic))
{
// Attempt to resolve response text with data store
for (const auto& infoEntry : dialogue->mInfo)
{
if (infoEntry.mId == entry.mInfo)
{
// We can't use the script interpreter because the environment is null
entry.mText = std::regex_replace(
infoEntry.mResponse, std::regex("%PCName"), mContext->mPlayerBase.mName);
entry.mText = std::regex_replace(
entry.mText, std::regex("%PCRace"), mContext->mPlayerBase.mRace.toString());
entry.mText = std::regex_replace(entry.mText, std::regex("%Name"), entry.mActorName);
break;
}
}
// Attempt to resolve record type with data store
switch (dialogue->mType)
{
case ESM::Dialogue::Journal:
entry.mType = ESM::JournalEntry::Type_Journal;
if (auto it = mContext->mJournalEntries.find(entry.mText);
it != mContext->mJournalEntries.end())
{
entry.mDayOfMonth = it->second.mDayOfMonth;
entry.mMonth = it->second.mMonth;
entry.mDay = it->second.mDay;
}
else
{
// We don't have a date, so default to 16 Last Seed (Day 1)
entry.mDayOfMonth = 16;
entry.mMonth = 7;
entry.mDay = 1;
}
break;
case ESM::Dialogue::Topic:
entry.mType = ESM::JournalEntry::Type_Topic;
break;
default:
throw std::runtime_error(
"Unrecognized dialogue record type for topic " + entry.mTopic.toDebugString());
}
}
else
entry.mType = ESM::JournalEntry::Type_Topic;
mOrderedEntries.emplace_back(entry);
}
std::stable_sort(mOrderedEntries.begin(), mOrderedEntries.end(),
[](const ESM::JournalEntry& a, const ESM::JournalEntry& b) { return a.mDay < b.mDay; });
for (const auto& entry : mOrderedEntries)
{
esm.startRecord(ESM::REC_JOUR);
entry.save(esm);
esm.endRecord(ESM::REC_JOUR);
// Journal-type entries need a matching quest-type entry in OpenMW
if (entry.mType == ESM::JournalEntry::Type_Journal)
{
ESM::JournalEntry questEntry;
questEntry.mTopic = entry.mTopic;
questEntry.mInfo = entry.mInfo;
questEntry.mType = ESM::JournalEntry::Type_Quest;
esm.startRecord(ESM::REC_JOUR);
questEntry.save(esm);
esm.endRecord(ESM::REC_JOUR);
}
}
}
private:
std::vector<INFO> mInfos;
std::vector<ESM::JournalEntry> mOrderedEntries;
};
class ConvertDIAL : public Converter
@ -480,28 +599,43 @@ namespace ESSImport
public:
void read(ESM::ESMReader& esm) override
{
std::string id = esm.getHNString("NAME");
ESM::RefId refId = esm.getHNRefId("NAME");
mContext->mCurrentDialogueRefId = refId; // Stored for subsequent INFO records
DIAL dial;
dial.load(esm);
if (dial.mIndex > 0)
mDials[id] = dial;
mQuestStates[refId] = dial;
}
void write(ESM::ESMWriter& esm) override
{
for (auto it = mDials.begin(); it != mDials.end(); ++it)
for (const auto& [refId, questState] : mQuestStates)
{
int finished = 0;
if (const ESM::Dialogue* dialogue = mDataStore->get<ESM::Dialogue>().search(refId))
{
for (const auto& info : dialogue->mInfo)
{
if (info.mData.mJournalIndex == questState.mIndex)
{
// If this journal index is finished, the imported quest state should be also
finished = info.mQuestStatus == ESM::DialInfo::QuestStatus::QS_Finished ? 1 : 0;
break;
}
}
}
esm.startRecord(ESM::REC_QUES);
ESM::QuestState state;
state.mFinished = 0;
state.mState = it->second.mIndex;
state.mTopic = ESM::RefId::stringRefId(it->first);
state.mFinished = finished;
state.mState = questState.mIndex;
state.mTopic = refId;
state.save(esm);
esm.endRecord(ESM::REC_QUES);
}
}
private:
std::map<std::string, DIAL> mDials;
std::map<ESM::RefId, DIAL> mQuestStates;
};
class ConvertQUES : public Converter
@ -509,8 +643,8 @@ namespace ESSImport
public:
void read(ESM::ESMReader& esm) override
{
std::string id = esm.getHNString("NAME");
QUES quest;
quest.mName = esm.getHNString("NAME");
quest.load(esm);
}
};
@ -522,6 +656,63 @@ namespace ESSImport
{
JOUR journal;
journal.load(esm);
// Strip formatting tokens and tags
std::string cleanedText
= std::regex_replace(journal.mText, std::regex(R"(@|#|<FONT COLOR="9F0000">|<\/FONT><BR>|<P>)"), "");
std::istringstream stream(cleanedText);
std::string line, currentEntryText;
int currentHeaderDay = 0, currentHeaderDayOfMonth = 0, currentHeaderMonth = 0, lineNum = 0;
// Load localized month names from the data store
std::map<std::string, int> monthMap = { { getGMST("sMonthMorningstar"), 0 },
{ getGMST("sMonthSunsdawn"), 1 }, { getGMST("sMonthFirstseed"), 2 }, { getGMST("sMonthRainshand"), 3 },
{ getGMST("sMonthSecondseed"), 4 }, { getGMST("sMonthMidyear"), 5 }, { getGMST("sMonthSunsheight"), 6 },
{ getGMST("sMonthLastseed"), 7 }, { getGMST("sMonthHeartfire"), 8 }, { getGMST("sMonthFrostfall"), 9 },
{ getGMST("sMonthSunsdusk"), 10 }, { getGMST("sMonthEveningstar"), 11 } };
std::string sDay = getGMST("sDay");
std::regex headerPattern(R"((\d{1,2})\s+([A-Za-z']+\s?[A-Za-z]*)\s+\()" + sDay + R"(\s+(\d+)\))");
while (std::getline(stream, line))
{
++lineNum;
// All journal entries begin with a date header line
std::smatch matches;
if (std::regex_match(line, matches, headerPattern))
{
if (lineNum > 1)
{
// Save the current entry when we reach the next header
ESM::JournalEntry entry;
entry.mDayOfMonth = currentHeaderDayOfMonth;
entry.mMonth = currentHeaderMonth;
entry.mDay = currentHeaderDay;
mContext->mJournalEntries[currentEntryText] = entry;
}
currentHeaderDayOfMonth = std::stoi(matches[1].str());
currentHeaderMonth = monthMap[matches[2].str()];
currentHeaderDay = std::stoi(matches[3].str());
currentEntryText.clear();
}
// Multi-line entry text follows the date header line
else
{
// Build the entry text between the headers
if (!currentEntryText.empty())
currentEntryText += "\n";
currentEntryText += line;
}
}
// Save the final entry
ESM::JournalEntry lastEntry;
lastEntry.mDayOfMonth = currentHeaderDayOfMonth;
lastEntry.mMonth = currentHeaderMonth;
lastEntry.mDay = currentHeaderDay;
mContext->mJournalEntries[currentEntryText] = lastEntry;
}
};

48
apps/essimporter/importer.cpp
View File

@ -8,13 +8,9 @@
#include <osgDB/ReadFile>
#include <components/esm/defs.hpp>
#include <components/esm3/cellid.hpp>
#include <components/esm3/esmreader.hpp>
#include <components/esm3/esmwriter.hpp>
#include <components/esm3/player.hpp>
#include <components/esm3/savedgame.hpp>
#include <components/esm3/cellid.hpp>
#include <components/esm3/loadalch.hpp>
#include <components/esm3/loadarmo.hpp>
#include <components/esm3/loadclot.hpp>
@ -22,11 +18,14 @@
#include <components/esm3/loadlevlist.hpp>
#include <components/esm3/loadspel.hpp>
#include <components/esm3/loadweap.hpp>
#include <components/esm3/player.hpp>
#include <components/esm3/savedgame.hpp>
#include <components/loadinglistener/loadinglistener.hpp>
#include <components/misc/constants.hpp>
#include <components/toutf8/toutf8.hpp>
#include "apps/openmw/mwworld/esmstore.hpp"
#include "importercontext.hpp"
#include "converter.hpp"
@ -87,11 +86,12 @@ namespace
namespace ESSImport
{
Importer::Importer(
const std::filesystem::path& essfile, const std::filesystem::path& outfile, const std::string& encoding)
Importer::Importer(const std::filesystem::path& essfile, const std::filesystem::path& outfile,
const std::string& encoding, const std::filesystem::path& datafilesPath)
: mEssFile(essfile)
, mOutFile(outfile)
, mEncoding(encoding)
, mDataFilesPath(datafilesPath)
{
}
@ -259,12 +259,35 @@ namespace ESSImport
{
ToUTF8::Utf8Encoder encoder(ToUTF8::calculateEncoding(mEncoding));
ESM::ESMReader esm;
esm.open(mEssFile);
esm.setEncoder(&encoder);
Context context;
esm.open(mEssFile);
const ESM::Header& header = esm.getHeader();
// Build the data store
MWWorld::ESMStore dataStore;
std::vector<std::string> contentFiles;
for (const auto& master : header.mMaster)
{
contentFiles.push_back(master.name);
}
static Loading::Listener dummyListener;
int index = 0;
ESM::Dialogue* dialogue = nullptr;
for (const auto& mContentFile : contentFiles)
{
ESM::ESMReader lEsm;
lEsm.setEncoder(&encoder);
lEsm.setIndex(index);
lEsm.open(mDataFilesPath.string() + "/" + mContentFile);
dataStore.load(lEsm, &dummyListener, dialogue);
++index;
}
dataStore.setUp();
Context context;
context.mPlayerCellName = header.mGameData.mCurrentCell.toString();
const unsigned int recREFR = ESM::fourCC("REFR");
@ -320,6 +343,7 @@ namespace ESSImport
for (const auto& converter : converters)
{
converter.second->setContext(context);
converter.second->setDataStore(dataStore);
}
while (esm.hasMoreRecs())

5
apps/essimporter/importer.hpp
View File

@ -9,8 +9,8 @@ namespace ESSImport
class Importer
{
public:
Importer(
const std::filesystem::path& essfile, const std::filesystem::path& outfile, const std::string& encoding);
Importer(const std::filesystem::path& essfile, const std::filesystem::path& outfile,
const std::string& encoding, const std::filesystem::path& datafilesPath);
void run();
@ -20,6 +20,7 @@ namespace ESSImport
std::filesystem::path mEssFile;
std::filesystem::path mOutFile;
std::string mEncoding;
std::filesystem::path mDataFilesPath;
};
}

4
apps/essimporter/importercontext.hpp
View File

@ -3,9 +3,11 @@
#include <map>
#include <components/esm/refid.hpp>
#include <components/esm3/controlsstate.hpp>
#include <components/esm3/dialoguestate.hpp>
#include <components/esm3/globalmap.hpp>
#include <components/esm3/journalentry.hpp>
#include <components/esm3/loadcell.hpp>
#include <components/esm3/loadcrea.hpp>
#include <components/esm3/loadnpc.hpp>
@ -28,7 +30,9 @@ namespace ESSImport
ESM::NPC mPlayerBase;
std::string mCustomPlayerClassName;
ESM::RefId mCurrentDialogueRefId;
ESM::DialogueState mDialogueState;
std::map<std::string, ESM::JournalEntry> mJournalEntries;
ESM::ControlsState mControlsState;

3
apps/essimporter/importinfo.hpp
View File

@ -15,7 +15,10 @@ namespace ESSImport
struct INFO
{
ESM::RefId mTopic;
ESM::RefId mInfo;
// Treated as a string by ESM::JournalEntry
std::string mActorRefId;
void load(ESM::ESMReader& esm);

4
apps/essimporter/main.cpp
View File

@ -20,6 +20,7 @@ Allowed options)");
addOption("help,h", "produce help message");
addOption("mwsave,m", bpo::value<Files::MaybeQuotedPath>(), "morrowind .ess save file");
addOption("output,o", bpo::value<Files::MaybeQuotedPath>(), "output file (.omwsave)");
addOption("dataFilePath,d", bpo::value<Files::MaybeQuotedPath>(), "path to Morrowind data files directory");
addOption("compare,c", "compare two .ess files");
addOption("encoding", boost::program_options::value<std::string>()->default_value("win1252"),
"encoding of the save file");
@ -46,8 +47,9 @@ Allowed options)");
const auto& essFile = variables["mwsave"].as<Files::MaybeQuotedPath>();
const auto& outputFile = variables["output"].as<Files::MaybeQuotedPath>();
std::string encoding = variables["encoding"].as<std::string>();
const auto& dataFilePath = variables["dataFilePath"].as<Files::MaybeQuotedPath>();
ESSImport::Importer importer(essFile, outputFile, encoding);
ESSImport::Importer importer(essFile, outputFile, encoding, dataFilePath);
if (variables.count("compare"))
importer.compare();