diff --git a/OTRExporter/OTRExporter/DisplayListExporter.cpp b/OTRExporter/OTRExporter/DisplayListExporter.cpp index ecd8faadb..dafa8eb72 100644 --- a/OTRExporter/OTRExporter/DisplayListExporter.cpp +++ b/OTRExporter/OTRExporter/DisplayListExporter.cpp @@ -209,7 +209,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina break; case G_MTX: { - if ((!Globals::Instance->HasSegment(GETSEGNUM(data))) || ((data & 0xFFFFFFFF) == 0x07000000)) // En_Zf and En_Ny place a DL in segment 7 + if ((!Globals::Instance->HasSegment(GETSEGNUM(data), res->parent->workerID)) || ((data & 0xFFFFFFFF) == 0x07000000)) // En_Zf and En_Ny place a DL in segment 7 { uint32_t pp = (data & 0x000000FF00000000) >> 32; uint32_t mm = (data & 0x00000000FFFFFFFF); @@ -404,7 +404,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina //case G_BRANCH_Z: case G_DL: { - if ((!Globals::Instance->HasSegment(GETSEGNUM(data)) && (int)opF3D != G_BRANCH_Z) + if ((!Globals::Instance->HasSegment(GETSEGNUM(data), res->parent->workerID) && (int)opF3D != G_BRANCH_Z) || ((data & 0xFFFFFFFF) == 0x07000000)) // En_Zf and En_Ny place a DL in segment 7 { int32_t pp = (data & 0x00FF000000000000) >> 56; @@ -681,7 +681,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina uint32_t seg = data & 0xFFFFFFFF; int32_t texAddress = Seg2Filespace(data, dList->parent->baseAddress); - if (!Globals::Instance->HasSegment(GETSEGNUM(seg))) + if (!Globals::Instance->HasSegment(GETSEGNUM(seg), res->parent->workerID)) { int32_t __ = (data & 0x00FF000000000000) >> 48; int32_t www = (data & 0x00000FFF00000000) >> 32; @@ -699,7 +699,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina else { std::string texName = ""; - bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, dList->parent, "", texName); + bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, dList->parent, "", texName, res->parent->workerID); int32_t __ = (data & 0x00FF000000000000) >> 48; int32_t www = (data & 0x00000FFF00000000) >> 32; @@ -718,7 +718,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina if (foundDecl) { - ZFile* assocFile = Globals::Instance->GetSegment(GETSEGNUM(seg)); + ZFile* assocFile = Globals::Instance->GetSegment(GETSEGNUM(seg), res->parent->workerID); std::string assocFileName = assocFile->GetName(); std::string fName = ""; @@ -765,8 +765,6 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina auto segOffset = GETSEGOFFSET(addr); Declaration* vtxDecl = dList->parent->GetDeclarationRanged(segOffset); - //std::string vtxName = ""; - //bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, dList->parent, "", vtxName); int32_t aa = (data & 0x000000FF00000000ULL) >> 32; int32_t nn = (data & 0x000FF00000000000ULL) >> 44; diff --git a/OTRExporter/OTRExporter/RoomExporter.cpp b/OTRExporter/OTRExporter/RoomExporter.cpp index 1890711d9..5b7ce4323 100644 --- a/OTRExporter/OTRExporter/RoomExporter.cpp +++ b/OTRExporter/OTRExporter/RoomExporter.cpp @@ -407,7 +407,7 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite { uint32_t seg = cmdHeaders->headers[i] & 0xFFFFFFFF; std::string headerName = ""; - bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, room->parent, "", headerName); + bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, room->parent, "", headerName, res->parent->workerID); if (headerName == "NULL") writer->Write(""); else @@ -443,7 +443,7 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite SetCutscenes* cmdSetCutscenes = (SetCutscenes*)cmd; std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdSetCutscenes->cmdArg2, room->parent, "CutsceneData", listName); + Globals::Instance->GetSegmentedPtrName(cmdSetCutscenes->cmdArg2, room->parent, "CutsceneData", listName, res->parent->workerID); std::string fName = OTRExporter_DisplayList::GetPathToRes(room, listName); //std::string fName = StringHelper::Sprintf("%s\\%s", OTRExporter_DisplayList::GetParentFolderName(room).c_str(), listName.c_str()); writer->Write(fName); diff --git a/OTRExporter/OTRExporter/SkeletonExporter.cpp b/OTRExporter/OTRExporter/SkeletonExporter.cpp index cd9f7a8fd..3e0c58d06 100644 --- a/OTRExporter/OTRExporter/SkeletonExporter.cpp +++ b/OTRExporter/OTRExporter/SkeletonExporter.cpp @@ -23,7 +23,7 @@ void OTRExporter_Skeleton::Save(ZResource* res, const fs::path& outPath, BinaryW Declaration* skelDecl = skel->parent->GetDeclarationRanged(GETSEGOFFSET(skel->limbsTable.limbsAddresses[i])); std::string name; - bool foundDecl = Globals::Instance->GetSegmentedPtrName(skel->limbsTable.limbsAddresses[i], skel->parent, "", name); + bool foundDecl = Globals::Instance->GetSegmentedPtrName(skel->limbsTable.limbsAddresses[i], skel->parent, "", name, res->parent->workerID); if (foundDecl) { if (name.at(0) == '&') diff --git a/OTRExporter/OTRExporter/SkeletonLimbExporter.cpp b/OTRExporter/OTRExporter/SkeletonLimbExporter.cpp index d22c3d000..e29c9a425 100644 --- a/OTRExporter/OTRExporter/SkeletonLimbExporter.cpp +++ b/OTRExporter/OTRExporter/SkeletonLimbExporter.cpp @@ -86,7 +86,7 @@ void OTRExporter_SkeletonLimb::Save(ZResource* res, const fs::path& outPath, Bin if (limb->childPtr != 0) { std::string name; - bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->childPtr, limb->parent, "", name); + bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->childPtr, limb->parent, "", name, res->parent->workerID); if (foundDecl) { if (name.at(0) == '&') @@ -107,7 +107,7 @@ void OTRExporter_SkeletonLimb::Save(ZResource* res, const fs::path& outPath, Bin if (limb->siblingPtr != 0) { std::string name; - bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->siblingPtr, limb->parent, "", name); + bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->siblingPtr, limb->parent, "", name, res->parent->workerID); if (foundDecl) { if (name.at(0) == '&') @@ -128,7 +128,7 @@ void OTRExporter_SkeletonLimb::Save(ZResource* res, const fs::path& outPath, Bin if (limb->dListPtr != 0) { std::string name; - bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->dListPtr, limb->parent, "", name); + bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->dListPtr, limb->parent, "", name, res->parent->workerID); if (foundDecl) { if (name.at(0) == '&') @@ -149,7 +149,7 @@ void OTRExporter_SkeletonLimb::Save(ZResource* res, const fs::path& outPath, Bin if (limb->dList2Ptr != 0) { std::string name; - bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->dList2Ptr, limb->parent, "", name); + bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->dList2Ptr, limb->parent, "", name, res->parent->workerID); if (foundDecl) { if (name.at(0) == '&') diff --git a/OTRGui/src/game/game.cpp b/OTRGui/src/game/game.cpp index 0ea29b602..be8471c09 100644 --- a/OTRGui/src/game/game.cpp +++ b/OTRGui/src/game/game.cpp @@ -72,8 +72,15 @@ void OTRGame::init(){ } } -void ExtractRom() { - const WriteResult result = ExtractBaserom(patched_rom); +void ExtractRom() +{ + WriteResult result; + + if (oldExtractMode) + ExtractBaserom(patched_rom); + else + result.error = NULLSTR; + if (result.error == NULLSTR) { if (MoonUtils::exists("oot.otr")) MoonUtils::rm("oot.otr"); if (MoonUtils::exists("Extract")) MoonUtils::rm("Extract"); diff --git a/OTRGui/src/impl/extractor/extractor.cpp b/OTRGui/src/impl/extractor/extractor.cpp index 1196733f7..e28fa5513 100644 --- a/OTRGui/src/impl/extractor/extractor.cpp +++ b/OTRGui/src/impl/extractor/extractor.cpp @@ -112,7 +112,7 @@ void startWorker(RomVersion version) { else { std::string execStr = Util::format("assets/extractor/%s", isWindows() ? "ZAPD.exe" : "ZAPD.out"); - std::string args = Util::format(" ed -eh -i %s -b tmp/baserom/ -o %s -osf %s -gsf 1 -rconf assets/extractor/Config_%s.xml -se OTR %s", path.c_str(), path + "../", path + "../", GetXMLVersion(version).c_str(), ""); + std::string args = Util::format(" ed -eh -i %s -b tmp/rom.z64 -fl assets/extractor/filelists -o %s -osf %s -gsf 1 -rconf assets/extractor/Config_%s.xml -se OTR %s", path.c_str(), path + "../", path + "../", GetXMLVersion(version).c_str(), ""); ProcessResult result = NativeFS->LaunchProcess(execStr + args); if (result.exitCode != 0) { diff --git a/ZAPDTR/ZAPD/FileWorker.cpp b/ZAPDTR/ZAPD/FileWorker.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/ZAPDTR/ZAPD/FileWorker.h b/ZAPDTR/ZAPD/FileWorker.h new file mode 100644 index 000000000..6ceae8076 --- /dev/null +++ b/ZAPDTR/ZAPD/FileWorker.h @@ -0,0 +1,15 @@ +#pragma once + +#include +#include +#include +#include "ZFile.h" + +class FileWorker +{ +public: + std::vector files; + std::vector externalFiles; + std::vector segments; + std::map> segmentRefFiles; +}; \ No newline at end of file diff --git a/ZAPDTR/ZAPD/Globals.cpp b/ZAPDTR/ZAPD/Globals.cpp index ae172b127..f0df0e547 100644 --- a/ZAPDTR/ZAPD/Globals.cpp +++ b/ZAPDTR/ZAPD/Globals.cpp @@ -34,30 +34,88 @@ Globals::~Globals() } } -void Globals::AddSegment(int32_t segment, ZFile* file) +void Globals::AddSegment(int32_t segment, ZFile* file, int workerID) { - if (std::find(segments.begin(), segments.end(), segment) == segments.end()) - segments.push_back(segment); - if (cfg.segmentRefFiles.find(segment) == cfg.segmentRefFiles.end()) - cfg.segmentRefFiles[segment] = std::vector(); - - cfg.segmentRefFiles[segment].push_back(file); -} - -bool Globals::HasSegment(int32_t segment) -{ - return std::find(segments.begin(), segments.end(), segment) != segments.end(); -} - -ZFile* Globals::GetSegment(int32_t segment) -{ - if (HasSegment(segment)) + if (!Globals::Instance->singleThreaded) { - int idx = std::find(segments.begin(), segments.end(), segment) - segments.begin(); - return files[idx]; + auto worker = workerData[workerID]; + + if (std::find(worker->segments.begin(), worker->segments.end(), segment) == + worker->segments.end()) + worker->segments.push_back(segment); + if (worker->segmentRefFiles.find(segment) == worker->segmentRefFiles.end()) + worker->segmentRefFiles[segment] = std::vector(); + + worker->segmentRefFiles[segment].push_back(file); } else - return nullptr; + { + if (std::find(segments.begin(), segments.end(), segment) == segments.end()) + segments.push_back(segment); + if (cfg.segmentRefFiles.find(segment) == cfg.segmentRefFiles.end()) + cfg.segmentRefFiles[segment] = std::vector(); + + cfg.segmentRefFiles[segment].push_back(file); + } +} + +bool Globals::HasSegment(int32_t segment, int workerID) +{ + if (!Globals::Instance->singleThreaded) + return std::find(workerData[workerID]->segments.begin(), + workerData[workerID]->segments.end(), segment) != workerData[workerID]->segments.end(); + else + return std::find(segments.begin(), segments.end(), segment) != segments.end(); +} + +ZFile* Globals::GetSegment(int32_t segment, int workerID) +{ + if (!Globals::Instance->singleThreaded) + { + if (HasSegment(segment, workerID)) + { + int idx = std::find(workerData[workerID]->segments.begin(), + workerData[workerID]->segments.end(), segment) - + workerData[workerID]->segments.begin(); + return workerData[workerID]->files[idx]; + } + else + return nullptr; + } + else + { + if (HasSegment(segment, workerID)) + { + int idx = std::find(segments.begin(), segments.end(), segment) - segments.begin(); + return files[idx]; + } + else + return nullptr; + } +} + +std::map> Globals::GetSegmentRefFiles(int workerID) +{ + if (!Globals::Instance->singleThreaded) + return workerData[workerID]->segmentRefFiles; + else + return cfg.segmentRefFiles; +} + +void Globals::AddFile(ZFile* file, int workerID) +{ + if (singleThreaded) + files.push_back(file); + else + workerData[workerID]->files.push_back(file); +} + +void Globals::AddExternalFile(ZFile* file, int workerID) +{ + if (singleThreaded) + externalFiles.push_back(file); + else + workerData[workerID]->externalFiles.push_back(file); } std::map& Globals::GetExporterMap() @@ -96,13 +154,19 @@ ExporterSet* Globals::GetExporterSet() std::vector Globals::GetBaseromFile(std::string fileName) { if (fileMode == ZFileMode::ExtractDirectory) - return rom->GetFile(StringHelper::Split(fileName, "baserom/")[1]); + { + if (StringHelper::Contains(fileName, "baserom/")) + fileName = StringHelper::Split(fileName, "baserom/")[1]; + + return rom->GetFile(fileName); + + } else return File::ReadAllBytes(fileName); } bool Globals::GetSegmentedPtrName(segptr_t segAddress, ZFile* currentFile, - const std::string& expectedType, std::string& declName) + const std::string& expectedType, std::string& declName, int workerID) { if (segAddress == 0) { @@ -138,9 +202,11 @@ bool Globals::GetSegmentedPtrName(segptr_t segAddress, ZFile* currentFile, if (currentFile->GetDeclarationPtrName(segAddress, expectedType, declName)) return true; } - else if (HasSegment(segment)) + else if (HasSegment(segment, workerID)) { - for (auto file : cfg.segmentRefFiles[segment]) + // OTRTODO: Multithreading + auto segs = GetSegmentRefFiles(workerID); + for (auto file : segs[segment]) { offset = Seg2Filespace(segAddress, file->baseAddress); @@ -184,7 +250,7 @@ bool Globals::GetSegmentedPtrName(segptr_t segAddress, ZFile* currentFile, bool Globals::GetSegmentedArrayIndexedName(segptr_t segAddress, size_t elementSize, ZFile* currentFile, const std::string& expectedType, - std::string& declName) + std::string& declName, int workerID) { if (segAddress == 0) { @@ -201,9 +267,11 @@ bool Globals::GetSegmentedArrayIndexedName(segptr_t segAddress, size_t elementSi if (addressFound) return true; } - else if (HasSegment(segment)) + else if (HasSegment(segment, workerID)) { - for (auto file : cfg.segmentRefFiles[segment]) + // OTRTODO: Multithreading + auto segs = GetSegmentRefFiles(workerID); + for (auto file : segs[segment]) { if (file->IsSegmentedInFilespaceRange(segAddress)) { diff --git a/ZAPDTR/ZAPD/Globals.h b/ZAPDTR/ZAPD/Globals.h index e52540cbb..683fd6fcd 100644 --- a/ZAPDTR/ZAPD/Globals.h +++ b/ZAPDTR/ZAPD/Globals.h @@ -6,6 +6,7 @@ #include "GameConfig.h" #include "ZFile.h" #include +#include class ZRoom; @@ -51,9 +52,10 @@ public: bool outputCrc = false; bool profile; // Measure performance of certain operations bool useLegacyZDList; + bool singleThreaded; VerbosityLevel verbosity; // ZAPD outputs additional information ZFileMode fileMode; - fs::path baseRomPath, inputPath, outputPath, sourceOutputPath, cfgPath; + fs::path baseRomPath, inputPath, outputPath, sourceOutputPath, cfgPath, fileListPath; TextureType texType; ZGame game; GameConfig cfg; @@ -68,6 +70,8 @@ public: std::vector externalFiles; std::vector segments; + std::map workerData; + std::string currentExporter; static std::map& GetExporterMap(); static void AddExporter(std::string exporterName, ExporterSet* exporterSet); @@ -75,9 +79,12 @@ public: Globals(); ~Globals(); - void AddSegment(int32_t segment, ZFile* file); - bool HasSegment(int32_t segment); - ZFile* GetSegment(int32_t segment); + void AddSegment(int32_t segment, ZFile* file, int workerID); + bool HasSegment(int32_t segment, int workerID); + ZFile* GetSegment(int32_t segment, int workerID); + std::map> GetSegmentRefFiles(int workerID); + void AddFile(ZFile* file, int workerID); + void AddExternalFile(ZFile* file, int workerID); ZResourceExporter* GetExporter(ZResourceType resType); ExporterSet* GetExporterSet(); @@ -93,8 +100,8 @@ public: * in which case `declName` will be set to the address formatted as a pointer. */ bool GetSegmentedPtrName(segptr_t segAddress, ZFile* currentFile, - const std::string& expectedType, std::string& declName); + const std::string& expectedType, std::string& declName, int workerID); bool GetSegmentedArrayIndexedName(segptr_t segAddress, size_t elementSize, ZFile* currentFile, - const std::string& expectedType, std::string& declName); + const std::string& expectedType, std::string& declName, int workerID); }; diff --git a/ZAPDTR/ZAPD/Main.cpp b/ZAPDTR/ZAPD/Main.cpp index dbfaf1944..194abd3db 100644 --- a/ZAPDTR/ZAPD/Main.cpp +++ b/ZAPDTR/ZAPD/Main.cpp @@ -23,16 +23,20 @@ #include #include #include "tinyxml2.h" +#include //extern const char gBuildHash[]; const char gBuildHash[] = ""; bool Parse(const fs::path& xmlFilePath, const fs::path& basePath, const fs::path& outPath, - ZFileMode fileMode); + ZFileMode fileMode, int workerID); void BuildAssetTexture(const fs::path& pngFilePath, TextureType texType, const fs::path& outPath); void BuildAssetBackground(const fs::path& imageFilePath, const fs::path& outPath); void BuildAssetBlob(const fs::path& blobFilePath, const fs::path& outPath); +int ExtractFunc(int workerID, int fileListSize, std::string fileListItem, ZFileMode fileMode); + +volatile int numWorkersLeft = 0; #if !defined(_MSC_VER) && !defined(__CYGWIN__) #define ARRAY_COUNT(arr) (sizeof(arr) / sizeof(arr[0])) @@ -182,6 +186,10 @@ int main(int argc, char* argv[]) { Globals::Instance->cfgPath = argv[++i]; } + else if (arg == "-fl") // Set baserom filelist path + { + Globals::Instance->fileListPath = argv[++i]; + } else if (arg == "-rconf") // Read Config File { Globals::Instance->cfg.ReadConfigFile(argv[++i]); @@ -254,7 +262,7 @@ int main(int argc, char* argv[]) Globals::Instance->fileMode = fileMode; if (fileMode == ZFileMode::ExtractDirectory) - Globals::Instance->rom = new ZRom("baserom.z64"); + Globals::Instance->rom = new ZRom(Globals::Instance->baseRomPath.string()); // We've parsed through our commands once. If an exporter exists, it's been set by now. // Now we'll parse through them again but pass them on to our exporter if one is available. @@ -281,7 +289,6 @@ int main(int argc, char* argv[]) if (exporterSet != nullptr && exporterSet->processFileModeFunc != nullptr) procFileModeSuccess = exporterSet->processFileModeFunc(fileMode); - if (!procFileModeSuccess) { if (fileMode == ZFileMode::ExtractDirectory) @@ -289,49 +296,44 @@ int main(int argc, char* argv[]) std::vector fileList = Directory::ListFiles(Globals::Instance->inputPath.string()); + const int num_threads = std::thread::hardware_concurrency(); + ctpl::thread_pool pool(num_threads / 2); + bool parseSuccessful; - auto start = std::chrono::steady_clock::now(); + int fileListSize = fileList.size(); + Globals::Instance->singleThreaded = false; - for (int i = 0; i < fileList.size(); i++) + for (int i = 0; i < fileListSize; i++) + Globals::Instance->workerData[i] = new FileWorker(); + + numWorkersLeft = fileListSize; + + for (int i = 0; i < fileListSize; i++) { - printf("(%i / %i): %s\n", (i+1), fileList.size(), fileList[i].c_str()); - - for (auto& extFile : Globals::Instance->cfg.externalFiles) + if (Globals::Instance->singleThreaded) { - fs::path externalXmlFilePath = - Globals::Instance->cfg.externalXmlFolder / extFile.xmlPath; - - if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO) - { - printf("Parsing external file from config: '%s'\n", - externalXmlFilePath.c_str()); - } - - parseSuccessful = Parse(externalXmlFilePath, Globals::Instance->baseRomPath, - extFile.outPath, ZFileMode::ExternalFile); - - if (!parseSuccessful) - return 1; + ExtractFunc(i, fileList.size(), fileList[i], fileMode); } - - parseSuccessful = Parse(fileList[i], Globals::Instance->baseRomPath, - Globals::Instance->outputPath, fileMode); - - if (!parseSuccessful) - return 1; - - for (int i = 0; i < Globals::Instance->files.size(); i++) + else { - delete Globals::Instance->files[i]; - Globals::Instance->files.erase(Globals::Instance->files.begin() + i); - i--; + std::string fileListItem = fileList[i]; + pool.push([i, fileListSize, fileListItem, fileMode](int) { + ExtractFunc(i, fileListSize, fileListItem, fileMode); + }); } + } - Globals::Instance->externalFiles.clear(); - Globals::Instance->segments.clear(); - Globals::Instance->cfg.segmentRefFiles.clear(); + if (!Globals::Instance->singleThreaded) + { + while (true) + { + if (numWorkersLeft <= 0) + break; + + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + } } auto end = std::chrono::steady_clock::now(); @@ -339,7 +341,7 @@ int main(int argc, char* argv[]) std::chrono::duration_cast(end - start).count(); printf("Generated OTR File Data in %i seconds\n", diff); - } + } else { bool parseSuccessful; @@ -356,7 +358,7 @@ int main(int argc, char* argv[]) } parseSuccessful = Parse(externalXmlFilePath, Globals::Instance->baseRomPath, - extFile.outPath, ZFileMode::ExternalFile); + extFile.outPath, ZFileMode::ExternalFile, 0); if (!parseSuccessful) return 1; @@ -364,7 +366,7 @@ int main(int argc, char* argv[]) parseSuccessful = Parse(Globals::Instance->inputPath, Globals::Instance->baseRomPath, - Globals::Instance->outputPath, fileMode); + Globals::Instance->outputPath, fileMode, 0); if (!parseSuccessful) return 1; } @@ -403,8 +405,68 @@ int main(int argc, char* argv[]) return 0; } +int ExtractFunc(int workerID, int fileListSize, std::string fileListItem, ZFileMode fileMode) +{ + bool parseSuccessful; + + printf("(%i / %i): %s\n", (workerID + 1), fileListSize, fileListItem.c_str()); + + for (auto& extFile : Globals::Instance->cfg.externalFiles) + { + fs::path externalXmlFilePath = Globals::Instance->cfg.externalXmlFolder / extFile.xmlPath; + + if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO) + { + printf("Parsing external file from config: '%s'\n", externalXmlFilePath.c_str()); + } + + parseSuccessful = Parse(externalXmlFilePath, Globals::Instance->baseRomPath, + extFile.outPath, ZFileMode::ExternalFile, workerID); + + if (!parseSuccessful) + return 1; + } + + parseSuccessful = Parse(fileListItem, Globals::Instance->baseRomPath, + Globals::Instance->outputPath, fileMode, workerID); + + if (!parseSuccessful) + return 1; + + if (Globals::Instance->singleThreaded) + { + for (int i = 0; i < Globals::Instance->files.size(); i++) + { + delete Globals::Instance->files[i]; + Globals::Instance->files.erase(Globals::Instance->files.begin() + i); + i--; + } + + Globals::Instance->externalFiles.clear(); + Globals::Instance->segments.clear(); + Globals::Instance->cfg.segmentRefFiles.clear(); + } + else + { + for (int i = 0; i < Globals::Instance->workerData[workerID]->files.size(); i++) + { + delete Globals::Instance->workerData[workerID]->files[i]; + Globals::Instance->workerData[workerID]->files.erase( + Globals::Instance->workerData[workerID]->files.begin() + + i); + i--; + } + + Globals::Instance->workerData[workerID]->externalFiles.clear(); + Globals::Instance->workerData[workerID]->segments.clear(); + Globals::Instance->workerData[workerID]->segmentRefFiles.clear(); + + numWorkersLeft--; + } +} + bool Parse(const fs::path& xmlFilePath, const fs::path& basePath, const fs::path& outPath, - ZFileMode fileMode) + ZFileMode fileMode, int workerID) { tinyxml2::XMLDocument doc; tinyxml2::XMLError eResult = doc.LoadFile(xmlFilePath.string().c_str()); @@ -432,11 +494,11 @@ bool Parse(const fs::path& xmlFilePath, const fs::path& basePath, const fs::path { if (std::string_view(child->Name()) == "File") { - ZFile* file = new ZFile(fileMode, child, basePath, outPath, "", xmlFilePath); - Globals::Instance->files.push_back(file); + ZFile* file = new ZFile(fileMode, child, basePath, outPath, "", xmlFilePath, workerID); + Globals::Instance->AddFile(file, workerID); if (fileMode == ZFileMode::ExternalFile) { - Globals::Instance->externalFiles.push_back(file); + Globals::Instance->AddExternalFile(file, workerID); file->isExternalFile = true; } } @@ -469,7 +531,7 @@ bool Parse(const fs::path& xmlFilePath, const fs::path& basePath, const fs::path } // Recursion. What can go wrong? - Parse(externalXmlFilePath, basePath, externalOutFilePath, ZFileMode::ExternalFile); + Parse(externalXmlFilePath, basePath, externalOutFilePath, ZFileMode::ExternalFile, workerID); } else { @@ -488,7 +550,14 @@ bool Parse(const fs::path& xmlFilePath, const fs::path& basePath, const fs::path if (exporterSet != nullptr && exporterSet->beginXMLFunc != nullptr) exporterSet->beginXMLFunc(); - for (ZFile* file : Globals::Instance->files) + std::vector files; + + if (Globals::Instance->singleThreaded) + files = Globals::Instance->files; + else + files = Globals::Instance->workerData[workerID]->files; + + for (ZFile* file : files) { if (fileMode == ZFileMode::BuildSourceFile) file->BuildSourceFile(); diff --git a/ZAPDTR/ZAPD/OtherStructs/SkinLimbStructs.cpp b/ZAPDTR/ZAPD/OtherStructs/SkinLimbStructs.cpp index 8ce215c97..c9a537ced 100644 --- a/ZAPDTR/ZAPD/OtherStructs/SkinLimbStructs.cpp +++ b/ZAPDTR/ZAPD/OtherStructs/SkinLimbStructs.cpp @@ -199,8 +199,10 @@ std::string Struct_800A598C::GetBodySourceCode() const { std::string unk_8_Str; std::string unk_C_Str; - Globals::Instance->GetSegmentedPtrName(unk_8, parent, "Struct_800A57C0", unk_8_Str); - Globals::Instance->GetSegmentedPtrName(unk_C, parent, "Struct_800A598C_2", unk_C_Str); + Globals::Instance->GetSegmentedPtrName(unk_8, parent, "Struct_800A57C0", unk_8_Str, + parent->workerID); + Globals::Instance->GetSegmentedPtrName(unk_C, parent, "Struct_800A598C_2", unk_C_Str, + parent->workerID); std::string entryStr = StringHelper::Sprintf("\n\t\tARRAY_COUNTU(%s), ARRAY_COUNTU(%s),\n", unk_8_Str.c_str(), unk_C_Str.c_str()); @@ -316,8 +318,9 @@ std::string Struct_800A5E28::GetBodySourceCode() const { std::string unk_4_Str; std::string unk_8_Str; - Globals::Instance->GetSegmentedPtrName(unk_4, parent, "Struct_800A598C", unk_4_Str); - Globals::Instance->GetSegmentedPtrName(unk_8, parent, "Gfx", unk_8_Str); + Globals::Instance->GetSegmentedPtrName(unk_4, parent, "Struct_800A598C", unk_4_Str, + parent->workerID); + Globals::Instance->GetSegmentedPtrName(unk_8, parent, "Gfx", unk_8_Str, parent->workerID); std::string entryStr = "\n"; entryStr += StringHelper::Sprintf("\t%i, ARRAY_COUNTU(%s),\n", unk_0, unk_4_Str.c_str()); diff --git a/ZAPDTR/ZAPD/OutputFormatter.cpp b/ZAPDTR/ZAPD/OutputFormatter.cpp index dbb8692c0..7fac434b2 100644 --- a/ZAPDTR/ZAPD/OutputFormatter.cpp +++ b/ZAPDTR/ZAPD/OutputFormatter.cpp @@ -3,7 +3,7 @@ void OutputFormatter::Flush() { - //if (!Globals::Instance->otrMode) + //if (!Globals::Instance->otrMode) // OTRTODO: MULTITHREADING { if (col > lineLimit && !Globals::Instance->otrMode) { @@ -31,6 +31,10 @@ void OutputFormatter::Flush() int OutputFormatter::Write(const char* buf, int count) { + // OTRTODO + //if (!Globals::Instance->singleThreaded) + //return 0; + for (int i = 0; i < count; i++) { char c = buf[i]; @@ -92,7 +96,7 @@ int OutputFormatter::Write(const std::string& buf) return Write(buf.data(), buf.size()); } -OutputFormatter* OutputFormatter::Instance; +__declspec(thread) OutputFormatter* OutputFormatter::Instance; int OutputFormatter::WriteStatic(const char* buf, int count) { diff --git a/ZAPDTR/ZAPD/OutputFormatter.h b/ZAPDTR/ZAPD/OutputFormatter.h index 28955b1cd..f008df2cb 100644 --- a/ZAPDTR/ZAPD/OutputFormatter.h +++ b/ZAPDTR/ZAPD/OutputFormatter.h @@ -25,7 +25,7 @@ private: void Flush(); - static OutputFormatter* Instance; + static __declspec(thread) OutputFormatter* Instance; static int WriteStatic(const char* buf, int count); public: diff --git a/ZAPDTR/ZAPD/ZAPD.vcxproj b/ZAPDTR/ZAPD/ZAPD.vcxproj index f9532c6b8..91ff515a2 100644 --- a/ZAPDTR/ZAPD/ZAPD.vcxproj +++ b/ZAPDTR/ZAPD/ZAPD.vcxproj @@ -184,6 +184,7 @@ + @@ -273,7 +274,9 @@ + + diff --git a/ZAPDTR/ZAPD/ZAPD.vcxproj.filters b/ZAPDTR/ZAPD/ZAPD.vcxproj.filters index 0d1b20086..c122b7e56 100644 --- a/ZAPDTR/ZAPD/ZAPD.vcxproj.filters +++ b/ZAPDTR/ZAPD/ZAPD.vcxproj.filters @@ -294,6 +294,9 @@ Source Files\Yaz0 + + Source Files + @@ -560,6 +563,12 @@ Header Files\Yaz0 + + Header Files + + + Header Files\Libraries + diff --git a/ZAPDTR/ZAPD/ZAnimation.cpp b/ZAPDTR/ZAPD/ZAnimation.cpp index 3c3100850..ef4735e09 100644 --- a/ZAPDTR/ZAPD/ZAnimation.cpp +++ b/ZAPDTR/ZAPD/ZAnimation.cpp @@ -150,10 +150,11 @@ void ZNormalAnimation::DeclareReferences(const std::string& prefix) std::string ZNormalAnimation::GetBodySourceCode() const { std::string frameDataName; - Globals::Instance->GetSegmentedPtrName(rotationValuesSeg, parent, "s16", frameDataName); + Globals::Instance->GetSegmentedPtrName(rotationValuesSeg, parent, "s16", frameDataName, + parent->workerID); std::string jointIndicesName; Globals::Instance->GetSegmentedPtrName(rotationIndicesSeg, parent, "JointIndex", - jointIndicesName); + jointIndicesName, parent->workerID); std::string headerStr = StringHelper::Sprintf("\n\t{ %i }, %s,\n", frameCount, frameDataName.c_str()); @@ -190,7 +191,7 @@ void ZLinkAnimation::ParseRawData() std::string ZLinkAnimation::GetBodySourceCode() const { std::string segSymbol; - Globals::Instance->GetSegmentedPtrName(segmentAddress, parent, "", segSymbol); + Globals::Instance->GetSegmentedPtrName(segmentAddress, parent, "", segSymbol, parent->workerID); return StringHelper::Sprintf("\n\t{ %i }, %s\n", frameCount, segSymbol.c_str()); } @@ -390,12 +391,13 @@ void ZCurveAnimation::DeclareReferences(const std::string& prefix) std::string ZCurveAnimation::GetBodySourceCode() const { std::string refIndexStr; - Globals::Instance->GetSegmentedPtrName(refIndex, parent, "u8", refIndexStr); + Globals::Instance->GetSegmentedPtrName(refIndex, parent, "u8", refIndexStr, parent->workerID); std::string transformDataStr; Globals::Instance->GetSegmentedPtrName(transformData, parent, "TransformData", - transformDataStr); + transformDataStr, parent->workerID); std::string copyValuesStr; - Globals::Instance->GetSegmentedPtrName(copyValues, parent, "s16", copyValuesStr); + Globals::Instance->GetSegmentedPtrName(copyValues, parent, "s16", copyValuesStr, + parent->workerID); return StringHelper::Sprintf("\n\t%s,\n\t%s,\n\t%s,\n\t%i, %i\n", refIndexStr.c_str(), transformDataStr.c_str(), copyValuesStr.c_str(), unk_0C, unk_10); @@ -517,8 +519,10 @@ std::string ZLegacyAnimation::GetBodySourceCode() const std::string frameDataName; std::string jointKeyName; - Globals::Instance->GetSegmentedPtrName(frameData, parent, "s16", frameDataName); - Globals::Instance->GetSegmentedPtrName(jointKey, parent, "JointKey", jointKeyName); + Globals::Instance->GetSegmentedPtrName(frameData, parent, "s16", frameDataName, + parent->workerID); + Globals::Instance->GetSegmentedPtrName(jointKey, parent, "JointKey", jointKeyName, + parent->workerID); body += StringHelper::Sprintf("\t%i, %i,\n", frameCount, limbCount); body += StringHelper::Sprintf("\t%s,\n", frameDataName.c_str()); diff --git a/ZAPDTR/ZAPD/ZCollision.cpp b/ZAPDTR/ZAPD/ZCollision.cpp index 5737783e3..2a282cf2f 100644 --- a/ZAPDTR/ZAPD/ZCollision.cpp +++ b/ZAPDTR/ZAPD/ZCollision.cpp @@ -198,23 +198,27 @@ std::string ZCollisionHeader::GetBodySourceCode() const declaration += StringHelper::Sprintf("\t{ %i, %i, %i },\n", absMaxX, absMaxY, absMaxZ); std::string vtxName; - Globals::Instance->GetSegmentedPtrName(vtxAddress, parent, "Vec3s", vtxName); + Globals::Instance->GetSegmentedPtrName(vtxAddress, parent, "Vec3s", vtxName, parent->workerID); declaration += StringHelper::Sprintf("\t%i,\n\t%s,\n", numVerts, vtxName.c_str()); std::string polyName; - Globals::Instance->GetSegmentedPtrName(polyAddress, parent, "CollisionPoly", polyName); + Globals::Instance->GetSegmentedPtrName(polyAddress, parent, "CollisionPoly", polyName, + parent->workerID); declaration += StringHelper::Sprintf("\t%i,\n\t%s,\n", numPolygons, polyName.c_str()); std::string surfaceName; - Globals::Instance->GetSegmentedPtrName(polyTypeDefAddress, parent, "SurfaceType", surfaceName); + Globals::Instance->GetSegmentedPtrName(polyTypeDefAddress, parent, "SurfaceType", surfaceName, + parent->workerID); declaration += StringHelper::Sprintf("\t%s,\n", surfaceName.c_str()); std::string camName; - Globals::Instance->GetSegmentedPtrName(camDataAddress, parent, "CamData", camName); + Globals::Instance->GetSegmentedPtrName(camDataAddress, parent, "CamData", camName, + parent->workerID); declaration += StringHelper::Sprintf("\t%s,\n", camName.c_str()); std::string waterBoxName; - Globals::Instance->GetSegmentedPtrName(waterBoxAddress, parent, "WaterBox", waterBoxName); + Globals::Instance->GetSegmentedPtrName(waterBoxAddress, parent, "WaterBox", waterBoxName, + parent->workerID); declaration += StringHelper::Sprintf("\t%i,\n\t%s\n", numWaterBoxes, waterBoxName.c_str()); return declaration; diff --git a/ZAPDTR/ZAPD/ZDisplayList.cpp b/ZAPDTR/ZAPD/ZDisplayList.cpp index 0cf1d5458..93269b65e 100644 --- a/ZAPDTR/ZAPD/ZDisplayList.cpp +++ b/ZAPDTR/ZAPD/ZDisplayList.cpp @@ -553,7 +553,8 @@ int32_t ZDisplayList::OptimizationCheck_LoadTextureBlock(int32_t startIndex, std lastTexSeg = segmentNumber; - Globals::Instance->GetSegmentedPtrName(data & 0xFFFFFFFF, parent, "", texStr); + Globals::Instance->GetSegmentedPtrName(data & 0xFFFFFFFF, parent, "", texStr, + parent->workerID); } // gsDPSetTile @@ -705,7 +706,7 @@ void ZDisplayList::Opcode_G_DL(uint64_t data, const std::string& prefix, char* l if (pp != 0) { - if (!Globals::Instance->HasSegment(segNum)) + if (!Globals::Instance->HasSegment(segNum, parent->workerID)) sprintf(line, "gsSPBranchList(0x%08" PRIX64 "),", data & 0xFFFFFFFF); else if (dListDecl != nullptr) sprintf(line, "gsSPBranchList(%s),", dListDecl->varName.c_str()); @@ -715,7 +716,7 @@ void ZDisplayList::Opcode_G_DL(uint64_t data, const std::string& prefix, char* l } else { - if (!Globals::Instance->HasSegment(segNum)) + if (!Globals::Instance->HasSegment(segNum, parent->workerID)) sprintf(line, "gsSPDisplayList(0x%08" PRIX64 "),", data & 0xFFFFFFFF); else if (dListDecl != nullptr) sprintf(line, "gsSPDisplayList(%s),", dListDecl->varName.c_str()); @@ -726,7 +727,7 @@ void ZDisplayList::Opcode_G_DL(uint64_t data, const std::string& prefix, char* l // if (segNum == 8 || segNum == 9 || segNum == 10 || segNum == 11 || segNum == 12 || segNum == // 13) // Used for runtime-generated display lists - if (!Globals::Instance->HasSegment(segNum)) + if (!Globals::Instance->HasSegment(segNum, parent->workerID)) { if (pp != 0) sprintf(line, "gsSPBranchList(0x%08" PRIX64 "),", data & 0xFFFFFFFF); @@ -847,7 +848,7 @@ void ZDisplayList::Opcode_G_VTX(uint64_t data, char* line) } // Hack: Don't extract vertices from a unknown segment. - if (!Globals::Instance->HasSegment(GETSEGNUM(data))) + if (!Globals::Instance->HasSegment(GETSEGNUM(data), parent->workerID)) { segptr_t segmented = data & 0xFFFFFFFF; references.push_back(segmented); @@ -951,7 +952,7 @@ void ZDisplayList::Opcode_G_SETTIMG(uint64_t data, const std::string& prefix, ch if (parent != nullptr) { - if (Globals::Instance->HasSegment(segmentNumber)) + if (Globals::Instance->HasSegment(segmentNumber, parent->workerID)) texDecl = parent->GetDeclaration(texAddress); else texDecl = parent->GetDeclaration(data); @@ -959,7 +960,7 @@ void ZDisplayList::Opcode_G_SETTIMG(uint64_t data, const std::string& prefix, ch if (texDecl != nullptr) sprintf(texStr, "%s", texDecl->varName.c_str()); - else if (data != 0 && Globals::Instance->HasSegment(segmentNumber)) + else if (data != 0 && Globals::Instance->HasSegment(segmentNumber, parent->workerID)) sprintf(texStr, "%sTex_%06X", prefix.c_str(), texAddress); else { @@ -972,7 +973,7 @@ void ZDisplayList::Opcode_G_SETTIMG(uint64_t data, const std::string& prefix, ch else { std::string texName; - Globals::Instance->GetSegmentedPtrName(data, parent, "", texName); + Globals::Instance->GetSegmentedPtrName(data, parent, "", texName, parent->workerID); sprintf(line, "gsDPSetTextureImage(%s, %s, %i, %s),", fmtTbl[fmt], sizTbl[siz], www + 1, texName.c_str()); } @@ -1672,7 +1673,7 @@ static int32_t GfxdCallback_Texture(segptr_t seg, int32_t fmt, int32_t siz, int3 self->TextureGenCheck(); std::string texName; - Globals::Instance->GetSegmentedPtrName(seg, self->parent, "", texName); + Globals::Instance->GetSegmentedPtrName(seg, self->parent, "", texName, self->parent->workerID); gfxd_puts(texName.c_str()); @@ -1696,7 +1697,7 @@ static int32_t GfxdCallback_Palette(uint32_t seg, [[maybe_unused]] int32_t idx, self->TextureGenCheck(); std::string palName; - Globals::Instance->GetSegmentedPtrName(seg, self->parent, "", palName); + Globals::Instance->GetSegmentedPtrName(seg, self->parent, "", palName, self->parent->workerID); gfxd_puts(palName.c_str()); @@ -1710,7 +1711,8 @@ static int32_t GfxdCallback_DisplayList(uint32_t seg) uint32_t dListSegNum = GETSEGNUM(seg); std::string dListName = ""; - bool addressFound = Globals::Instance->GetSegmentedPtrName(seg, self->parent, "Gfx", dListName); + bool addressFound = Globals::Instance->GetSegmentedPtrName(seg, self->parent, "Gfx", dListName, + self->parent->workerID); if (!addressFound && self->parent->segment == dListSegNum) { @@ -1733,7 +1735,8 @@ static int32_t GfxdCallback_Matrix(uint32_t seg) std::string mtxName; ZDisplayList* self = static_cast(gfxd_udata_get()); - bool addressFound = Globals::Instance->GetSegmentedPtrName(seg, self->parent, "Mtx", mtxName); + bool addressFound = Globals::Instance->GetSegmentedPtrName(seg, self->parent, "Mtx", mtxName, + self->parent->workerID); if (!addressFound && GETSEGNUM(seg) == self->parent->segment) { Declaration* decl = @@ -2004,7 +2007,7 @@ bool ZDisplayList::TextureGenCheck(int32_t texWidth, int32_t texHeight, uint32_t texWidth, texHeight, texIsPalette, texAddr); if ((texSeg != 0 || texAddr != 0) && texWidth > 0 && texHeight > 0 && texLoaded && - Globals::Instance->HasSegment(segmentNumber)) + Globals::Instance->HasSegment(segmentNumber, self->parent->workerID)) { ZFile* auxParent = nullptr; if (segmentNumber == self->parent->segment) @@ -2015,7 +2018,8 @@ bool ZDisplayList::TextureGenCheck(int32_t texWidth, int32_t texHeight, uint32_t { // Try to find a non-external file (i.e., one we are actually extracting) // which has the same segment number we are looking for. - for (auto& otherFile : Globals::Instance->cfg.segmentRefFiles[segmentNumber]) + auto segs = Globals::Instance->GetSegmentRefFiles(self->parent->workerID); + for (auto& otherFile : segs[segmentNumber]) { if (!otherFile->isExternalFile) { diff --git a/ZAPDTR/ZAPD/ZFile.cpp b/ZAPDTR/ZAPD/ZFile.cpp index f599e0144..b706c1914 100644 --- a/ZAPDTR/ZAPD/ZFile.cpp +++ b/ZAPDTR/ZAPD/ZFile.cpp @@ -41,6 +41,7 @@ ZFile::ZFile() baseAddress = 0; rangeStart = 0x000000000; rangeEnd = 0xFFFFFFFF; + workerID = 0; } ZFile::ZFile(const fs::path& nOutPath, const std::string& nName) : ZFile() @@ -51,7 +52,7 @@ ZFile::ZFile(const fs::path& nOutPath, const std::string& nName) : ZFile() } ZFile::ZFile(ZFileMode nMode, tinyxml2::XMLElement* reader, const fs::path& nBasePath, - const fs::path& nOutPath, const std::string& filename, const fs::path& nXmlFilePath) + const fs::path& nOutPath, const std::string& filename, const fs::path& nXmlFilePath, int nWorkerID) : ZFile() { xmlFilePath = nXmlFilePath; @@ -66,6 +67,7 @@ ZFile::ZFile(ZFileMode nMode, tinyxml2::XMLElement* reader, const fs::path& nBas outputPath = nOutPath; mode = nMode; + workerID = nWorkerID; ParseXML(reader, filename); if (mode != ZFileMode::ExternalFile) @@ -167,7 +169,7 @@ void ZFile::ParseXML(tinyxml2::XMLElement* reader, const std::string& filename) } } } - Globals::Instance->AddSegment(segment, this); + Globals::Instance->AddSegment(segment, this, workerID); if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO) { @@ -183,15 +185,20 @@ void ZFile::ParseXML(tinyxml2::XMLElement* reader, const std::string& filename) if (mode == ZFileMode::Extract || mode == ZFileMode::ExternalFile || mode == ZFileMode::ExtractDirectory) { - if (!File::Exists((basePath / name).string())) + if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory) { - std::string errorHeader = StringHelper::Sprintf("binary file '%s' does not exist.", - (basePath / name).c_str()); - HANDLE_ERROR_PROCESS(WarningType::Always, errorHeader, ""); + if (!File::Exists((basePath / name).string())) + { + std::string errorHeader = StringHelper::Sprintf("binary file '%s' does not exist.", + (basePath / name).c_str()); + HANDLE_ERROR_PROCESS(WarningType::Always, errorHeader, ""); + } } - //rawData = File::ReadAllBytes((basePath / name).string()); - rawData = Globals::Instance->GetBaseromFile((basePath / name).string()); + if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory) + rawData = Globals::Instance->GetBaseromFile(name); + else + rawData = Globals::Instance->GetBaseromFile((basePath / name).string()); if (reader->Attribute("RangeEnd") == nullptr) rangeEnd = rawData.size(); @@ -1093,7 +1100,7 @@ void ZFile::ProcessDeclarationText(Declaration* decl) { std::string vtxName; Globals::Instance->GetSegmentedArrayIndexedName(decl->references[refIndex], 0x10, this, - "Vtx", vtxName); + "Vtx", vtxName, workerID); decl->text.replace(i, 2, vtxName); refIndex++; diff --git a/ZAPDTR/ZAPD/ZFile.h b/ZAPDTR/ZAPD/ZFile.h index 235b682e9..7860aa4d3 100644 --- a/ZAPDTR/ZAPD/ZFile.h +++ b/ZAPDTR/ZAPD/ZFile.h @@ -35,6 +35,8 @@ public: std::string defines; std::vector resources; + int workerID; + // Default to using virtual addresses uint32_t segment = 0x80; uint32_t baseAddress, rangeStart, rangeEnd; @@ -42,7 +44,7 @@ public: ZFile(const fs::path& nOutPath, const std::string& nName); ZFile(ZFileMode nMode, tinyxml2::XMLElement* reader, const fs::path& nBasePath, - const fs::path& nOutPath, const std::string& filename, const fs::path& nXmlFilePath); + const fs::path& nOutPath, const std::string& filename, const fs::path& nXmlFilePath, int nWorkerID); ~ZFile(); std::string GetName() const; diff --git a/ZAPDTR/ZAPD/ZLimb.cpp b/ZAPDTR/ZAPD/ZLimb.cpp index 774fc7150..9eb7eac8d 100644 --- a/ZAPDTR/ZAPD/ZLimb.cpp +++ b/ZAPDTR/ZAPD/ZLimb.cpp @@ -223,16 +223,20 @@ std::string ZLimb::GetBodySourceCode() const std::string dListStr; std::string dListStr2; - Globals::Instance->GetSegmentedArrayIndexedName(dListPtr, 8, parent, "Gfx", dListStr); - Globals::Instance->GetSegmentedArrayIndexedName(dList2Ptr, 8, parent, "Gfx", dListStr2); + Globals::Instance->GetSegmentedArrayIndexedName(dListPtr, 8, parent, "Gfx", dListStr, + parent->workerID); + Globals::Instance->GetSegmentedArrayIndexedName(dList2Ptr, 8, parent, "Gfx", dListStr2, + parent->workerID); std::string entryStr = "\n\t"; if (type == ZLimbType::Legacy) { std::string childName; std::string siblingName; - Globals::Instance->GetSegmentedPtrName(childPtr, parent, "LegacyLimb", childName); - Globals::Instance->GetSegmentedPtrName(siblingPtr, parent, "LegacyLimb", siblingName); + Globals::Instance->GetSegmentedPtrName(childPtr, parent, "LegacyLimb", childName, + parent->workerID); + Globals::Instance->GetSegmentedPtrName(siblingPtr, parent, "LegacyLimb", siblingName, + parent->workerID); entryStr += StringHelper::Sprintf("%s,\n", dListStr.c_str()); entryStr += @@ -264,7 +268,8 @@ std::string ZLimb::GetBodySourceCode() const case ZLimbType::Skin: { std::string skinSegmentStr; - Globals::Instance->GetSegmentedPtrName(skinSegment, parent, "", skinSegmentStr); + Globals::Instance->GetSegmentedPtrName(skinSegment, parent, "", skinSegmentStr, + parent->workerID); entryStr += StringHelper::Sprintf("\t0x%02X, %s\n", skinSegmentType, skinSegmentStr.c_str()); } @@ -370,7 +375,7 @@ void ZLimb::DeclareDList(segptr_t dListSegmentedPtr, const std::string& prefix, std::string dlistName; bool declFound = Globals::Instance->GetSegmentedArrayIndexedName(dListSegmentedPtr, 8, parent, - "Gfx", dlistName); + "Gfx", dlistName, parent->workerID); if (declFound) return; diff --git a/ZAPDTR/ZAPD/ZPath.cpp b/ZAPDTR/ZAPD/ZPath.cpp index e19513db3..660821ce8 100644 --- a/ZAPDTR/ZAPD/ZPath.cpp +++ b/ZAPDTR/ZAPD/ZPath.cpp @@ -142,8 +142,8 @@ void PathwayEntry::DeclareReferences(const std::string& prefix) return; std::string pointsName; - bool addressFound = - Globals::Instance->GetSegmentedPtrName(listSegmentAddress, parent, "Vec3s", pointsName); + bool addressFound = Globals::Instance->GetSegmentedPtrName(listSegmentAddress, parent, "Vec3s", + pointsName, parent->workerID); if (addressFound) return; @@ -177,7 +177,8 @@ std::string PathwayEntry::GetBodySourceCode() const { std::string declaration; std::string listName; - Globals::Instance->GetSegmentedPtrName(listSegmentAddress, parent, "Vec3s", listName); + Globals::Instance->GetSegmentedPtrName(listSegmentAddress, parent, "Vec3s", listName, + parent->workerID); if (Globals::Instance->game == ZGame::MM_RETAIL) declaration += diff --git a/ZAPDTR/ZAPD/ZRom.cpp b/ZAPDTR/ZAPD/ZRom.cpp index 55167c596..2a1d3c8a1 100644 --- a/ZAPDTR/ZAPD/ZRom.cpp +++ b/ZAPDTR/ZAPD/ZRom.cpp @@ -7,6 +7,7 @@ #ifndef _MSC_VER #include #endif +#include namespace fs = std::filesystem; @@ -157,7 +158,7 @@ ZRom::ZRom(std::string romPath) break; } - auto path = StringHelper::Sprintf("CFG/filelists/%s", version.listPath.c_str()); + auto path = StringHelper::Sprintf("%s/%s", Globals::Instance->fileListPath.string().c_str(), version.listPath.c_str()); auto txt = File::ReadAllText(path); std::vector lines = StringHelper::Split(txt, "\n"); diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetActorCutsceneList.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetActorCutsceneList.cpp index 60391a9d1..4c9e8fb5f 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetActorCutsceneList.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetActorCutsceneList.cpp @@ -54,7 +54,8 @@ void SetActorCutsceneList::DeclareReferences(const std::string& prefix) std::string SetActorCutsceneList::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "ActorCutscene", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "ActorCutscene", listName, + parent->workerID); return StringHelper::Sprintf("SCENE_CMD_ACTOR_CUTSCENE_LIST(%i, %s)", cutscenes.size(), listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetActorList.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetActorList.cpp index 919d86f57..1cda3c2c6 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetActorList.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetActorList.cpp @@ -81,7 +81,8 @@ void SetActorList::DeclareReferencesLate(const std::string& prefix) std::string SetActorList::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "ActorEntry", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "ActorEntry", listName, + parent->workerID); if (numActors != actors.size()) { printf("%s: numActors(%i) ~ actors(%li)\n", parent->GetName().c_str(), numActors, diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetAlternateHeaders.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetAlternateHeaders.cpp index 629d4a0b0..a89c2b74d 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetAlternateHeaders.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetAlternateHeaders.cpp @@ -48,7 +48,8 @@ void SetAlternateHeaders::DeclareReferencesLate(const std::string& prefix) for (size_t i = 0; i < headers.size(); i++) { std::string altHeaderName; - Globals::Instance->GetSegmentedPtrName(headers.at(i), parent, "", altHeaderName); + Globals::Instance->GetSegmentedPtrName(headers.at(i), parent, "", altHeaderName, + parent->workerID); declaration += StringHelper::Sprintf("\t%s,", altHeaderName.c_str()); @@ -66,7 +67,8 @@ void SetAlternateHeaders::DeclareReferencesLate(const std::string& prefix) std::string SetAlternateHeaders::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "SceneCmd*", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "SceneCmd*", listName, + parent->workerID); return StringHelper::Sprintf("SCENE_CMD_ALTERNATE_HEADER_LIST(%s)", listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetAnimatedMaterialList.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetAnimatedMaterialList.cpp index 0b9a67e0e..73f72136c 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetAnimatedMaterialList.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetAnimatedMaterialList.cpp @@ -33,7 +33,8 @@ void SetAnimatedMaterialList::DeclareReferences(const std::string& prefix) std::string SetAnimatedMaterialList::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "AnimatedMaterial", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "AnimatedMaterial", listName, + parent->workerID); return StringHelper::Sprintf("SCENE_CMD_ANIMATED_MATERIAL_LIST(%s)", listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetCollisionHeader.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetCollisionHeader.cpp index 03aaa4bbb..4aafb7c70 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetCollisionHeader.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetCollisionHeader.cpp @@ -29,7 +29,8 @@ void SetCollisionHeader::DeclareReferences(const std::string& prefix) std::string SetCollisionHeader::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "CollisionHeader", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "CollisionHeader", listName, + parent->workerID); return StringHelper::Sprintf("SCENE_CMD_COL_HEADER(%s)", listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetCsCamera.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetCsCamera.cpp index 5cf0a3d03..577a89cd1 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetCsCamera.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetCsCamera.cpp @@ -71,7 +71,7 @@ void SetCsCamera::DeclareReferences(const std::string& prefix) { std::string camPointsName; Globals::Instance->GetSegmentedPtrName(cameras.at(0).GetCamAddress(), parent, "Vec3s", - camPointsName); + camPointsName, parent->workerID); std::string declaration; size_t index = 0; @@ -103,7 +103,8 @@ void SetCsCamera::DeclareReferences(const std::string& prefix) std::string SetCsCamera::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "CsCameraEntry", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "CsCameraEntry", listName, + parent->workerID); return StringHelper::Sprintf("SCENE_CMD_ACTOR_CUTSCENE_CAM_LIST(%i, %s)", cameras.size(), listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetCutscenes.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetCutscenes.cpp index e51e550b0..106f6f680 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetCutscenes.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetCutscenes.cpp @@ -86,7 +86,8 @@ void SetCutscenes::ParseRawData() std::string SetCutscenes::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "CutsceneData", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "CutsceneData", listName, + parent->workerID); if (Globals::Instance->game == ZGame::MM_RETAIL) return StringHelper::Sprintf("SCENE_CMD_CUTSCENE_LIST(%i, %s)", numCutscenes, diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetEntranceList.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetEntranceList.cpp index 8099bacf1..aa20102ff 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetEntranceList.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetEntranceList.cpp @@ -63,7 +63,8 @@ void SetEntranceList::DeclareReferencesLate([[maybe_unused]] const std::string& std::string SetEntranceList::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "EntranceEntry", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "EntranceEntry", listName, + parent->workerID); return StringHelper::Sprintf("SCENE_CMD_ENTRANCE_LIST(%s)", listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetExitList.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetExitList.cpp index ddc4c5d43..34d2a6e56 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetExitList.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetExitList.cpp @@ -58,7 +58,7 @@ void SetExitList::DeclareReferencesLate([[maybe_unused]] const std::string& pref std::string SetExitList::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "u16", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "u16", listName, parent->workerID); return StringHelper::Sprintf("SCENE_CMD_EXIT_LIST(%s)", listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetLightList.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetLightList.cpp index 2e023ff20..060fce0a2 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetLightList.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetLightList.cpp @@ -52,7 +52,8 @@ void SetLightList::DeclareReferences(const std::string& prefix) std::string SetLightList::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "LightInfo", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "LightInfo", listName, + parent->workerID); return StringHelper::Sprintf("SCENE_CMD_LIGHT_LIST(%i, %s)", numLights, listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetLightingSettings.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetLightingSettings.cpp index 08cd83d14..4824f3f61 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetLightingSettings.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetLightingSettings.cpp @@ -44,7 +44,8 @@ void SetLightingSettings::DeclareReferences(const std::string& prefix) std::string SetLightingSettings::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "LightSettings", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "LightSettings", listName, + parent->workerID); return StringHelper::Sprintf("SCENE_CMD_ENV_LIGHT_SETTINGS(%i, %s)", settings.size(), listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetMesh.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetMesh.cpp index ba0bbe2c2..0723fcf14 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetMesh.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetMesh.cpp @@ -78,7 +78,7 @@ std::string SetMesh::GenDListExterns(ZDisplayList* dList) std::string SetMesh::GetBodySourceCode() const { std::string list; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "", list); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "", list, parent->workerID); return StringHelper::Sprintf("SCENE_CMD_MESH(%s)", list.c_str()); } @@ -129,8 +129,8 @@ std::string PolygonDlist::GetBodySourceCode() const std::string bodyStr; std::string opaStr; std::string xluStr; - Globals::Instance->GetSegmentedPtrName(opa, parent, "Gfx", opaStr); - Globals::Instance->GetSegmentedPtrName(xlu, parent, "Gfx", xluStr); + Globals::Instance->GetSegmentedPtrName(opa, parent, "Gfx", opaStr, parent->workerID); + Globals::Instance->GetSegmentedPtrName(xlu, parent, "Gfx", xluStr, parent->workerID); if (polyType == 2) { @@ -294,7 +294,7 @@ std::string BgImage::GetBodySourceCode() const } std::string backgroundName; - Globals::Instance->GetSegmentedPtrName(source, parent, "", backgroundName); + Globals::Instance->GetSegmentedPtrName(source, parent, "", backgroundName, parent->workerID); bodyStr += StringHelper::Sprintf("%s, ", backgroundName.c_str()); bodyStr += "\n "; if (!isSubStruct) @@ -493,7 +493,7 @@ std::string PolygonType1::GetBodySourceCode() const bodyStr += StringHelper::Sprintf("%i, %i, ", type, format); std::string dlistStr; - Globals::Instance->GetSegmentedPtrName(dlist, parent, "", dlistStr); + Globals::Instance->GetSegmentedPtrName(dlist, parent, "", dlistStr, parent->workerID); bodyStr += StringHelper::Sprintf("%s, ", dlistStr.c_str()); bodyStr += "}, \n"; @@ -505,7 +505,7 @@ std::string PolygonType1::GetBodySourceCode() const bodyStr += single.GetBodySourceCode(); break; case 2: - Globals::Instance->GetSegmentedPtrName(list, parent, "BgImage", listStr); + Globals::Instance->GetSegmentedPtrName(list, parent, "BgImage", listStr, parent->workerID); bodyStr += StringHelper::Sprintf(" %i, %s, \n", count, listStr.c_str()); break; @@ -592,7 +592,7 @@ void PolygonType2::DeclareReferences(const std::string& prefix) std::string PolygonType2::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(start, parent, "", listName); + Globals::Instance->GetSegmentedPtrName(start, parent, "", listName, parent->workerID); std::string body = StringHelper::Sprintf("\n %i, %i,\n", type, polyDLists.size()); body += StringHelper::Sprintf(" %s,\n", listName.c_str()); diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetMinimapChests.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetMinimapChests.cpp index ec432b7d9..3e6c72a11 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetMinimapChests.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetMinimapChests.cpp @@ -50,7 +50,8 @@ void SetMinimapChests::DeclareReferences(const std::string& prefix) std::string SetMinimapChests::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "MinimapChest", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "MinimapChest", listName, + parent->workerID); return StringHelper::Sprintf("SCENE_CMD_MINIMAP_COMPASS_ICON_INFO(0x%02X, %s)", chests.size(), listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetMinimapList.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetMinimapList.cpp index be5d8f15d..255bd4e3a 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetMinimapList.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetMinimapList.cpp @@ -52,7 +52,8 @@ void SetMinimapList::DeclareReferences(const std::string& prefix) { std::string listName; - Globals::Instance->GetSegmentedPtrName(listSegmentAddr, parent, "MinimapEntry", listName); + Globals::Instance->GetSegmentedPtrName(listSegmentAddr, parent, "MinimapEntry", listName, + parent->workerID); std::string declaration = StringHelper::Sprintf("\n\t%s, 0x%08X\n", listName.c_str(), unk4); parent->AddDeclaration( @@ -65,7 +66,8 @@ void SetMinimapList::DeclareReferences(const std::string& prefix) std::string SetMinimapList::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "MinimapList", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "MinimapList", listName, + parent->workerID); return StringHelper::Sprintf("SCENE_CMD_MINIMAP_INFO(%s)", listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetObjectList.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetObjectList.cpp index fdd41e6cb..29519d3ce 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetObjectList.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetObjectList.cpp @@ -51,7 +51,7 @@ void SetObjectList::DeclareReferences(const std::string& prefix) std::string SetObjectList::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "s16", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "s16", listName, parent->workerID); return StringHelper::Sprintf("SCENE_CMD_OBJECT_LIST(%i, %s)", objects.size(), listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetPathways.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetPathways.cpp index 468aad822..f3bdb0872 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetPathways.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetPathways.cpp @@ -50,7 +50,7 @@ void SetPathways::DeclareReferencesLate(const std::string& prefix) std::string SetPathways::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "Path", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "Path", listName, parent->workerID); return StringHelper::Sprintf("SCENE_CMD_PATH_LIST(%s)", listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetRoomList.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetRoomList.cpp index 7027fa1f9..a63dd8772 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetRoomList.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetRoomList.cpp @@ -34,7 +34,7 @@ void SetRoomList::DeclareReferences(const std::string& prefix) std::string SetRoomList::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "RomFile", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "RomFile", listName, parent->workerID); return StringHelper::Sprintf("SCENE_CMD_ROOM_LIST(%i, %s)", romfile->rooms.size(), listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetStartPositionList.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetStartPositionList.cpp index f75b5e0d5..32a2b7000 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetStartPositionList.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetStartPositionList.cpp @@ -51,7 +51,8 @@ void SetStartPositionList::DeclareReferences(const std::string& prefix) std::string SetStartPositionList::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "ActorEntry", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "ActorEntry", listName, + parent->workerID); return StringHelper::Sprintf("SCENE_CMD_SPAWN_LIST(%i, %s)", actors.size(), listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZRoom/Commands/SetTransitionActorList.cpp b/ZAPDTR/ZAPD/ZRoom/Commands/SetTransitionActorList.cpp index a33d1c6e1..2414f0d2f 100644 --- a/ZAPDTR/ZAPD/ZRoom/Commands/SetTransitionActorList.cpp +++ b/ZAPDTR/ZAPD/ZRoom/Commands/SetTransitionActorList.cpp @@ -52,7 +52,8 @@ void SetTransitionActorList::DeclareReferences(const std::string& prefix) std::string SetTransitionActorList::GetBodySourceCode() const { std::string listName; - Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "TransitionActorEntry", listName); + Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "TransitionActorEntry", listName, + parent->workerID); return StringHelper::Sprintf("SCENE_CMD_TRANSITION_ACTOR_LIST(%i, %s)", transitionActors.size(), listName.c_str()); } diff --git a/ZAPDTR/ZAPD/ZSkeleton.cpp b/ZAPDTR/ZAPD/ZSkeleton.cpp index bab1f9111..bff4b8263 100644 --- a/ZAPDTR/ZAPD/ZSkeleton.cpp +++ b/ZAPDTR/ZAPD/ZSkeleton.cpp @@ -89,7 +89,8 @@ void ZSkeleton::DeclareReferences(const std::string& prefix) std::string ZSkeleton::GetBodySourceCode() const { std::string limbArrayName; - Globals::Instance->GetSegmentedPtrName(limbsArrayAddress, parent, "", limbArrayName); + Globals::Instance->GetSegmentedPtrName(limbsArrayAddress, parent, "", limbArrayName, + parent->workerID); switch (type) { @@ -245,7 +246,8 @@ std::string ZLimbTable::GetBodySourceCode() const for (size_t i = 0; i < count; i++) { std::string limbName; - Globals::Instance->GetSegmentedPtrName(limbsAddresses[i], parent, "", limbName); + Globals::Instance->GetSegmentedPtrName(limbsAddresses[i], parent, "", limbName, + parent->workerID); body += StringHelper::Sprintf("\t%s,", limbName.c_str()); if (i + 1 < count) diff --git a/ZAPDTR/ZAPD/ZText.cpp b/ZAPDTR/ZAPD/ZText.cpp index 5a8dad53c..58af61bc9 100644 --- a/ZAPDTR/ZAPD/ZText.cpp +++ b/ZAPDTR/ZAPD/ZText.cpp @@ -21,12 +21,12 @@ void ZText::ParseRawData() const auto& rawData = parent->GetRawData(); uint32_t currentPtr = StringHelper::StrToL(registeredAttributes.at("CodeOffset").value, 16); - std::vector codeData = Globals::Instance->GetBaseromFile(Globals::Instance->baseRomPath.string() + "code"); - //std::vector codeData = File::ReadAllBytes(Globals::Instance->baseRomPath.string() + "\\code"); + std::vector codeData; - // In some cases with the multi-process extractor it seems that it fails to read the code file if something else is reading from it at the same time. - //while (codeData.size() == 0) - //codeData = File::ReadAllBytes(Globals::Instance->baseRomPath.string() + "\\code"); + if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory) + codeData = Globals::Instance->GetBaseromFile("code"); + else + codeData = Globals::Instance->GetBaseromFile(Globals::Instance->baseRomPath.string() + "code"); while (true) { diff --git a/ZAPDTR/ZAPD/ZTextureAnimation.cpp b/ZAPDTR/ZAPD/ZTextureAnimation.cpp index 698054fa8..233a54dbb 100644 --- a/ZAPDTR/ZAPD/ZTextureAnimation.cpp +++ b/ZAPDTR/ZAPD/ZTextureAnimation.cpp @@ -349,9 +349,12 @@ std::string TextureColorChangingParams::GetBodySourceCode() const std::string envColorListName; std::string frameDataListName; - Globals::Instance->GetSegmentedPtrName(primColorListAddress, parent, "", primColorListName); - Globals::Instance->GetSegmentedPtrName(envColorListAddress, parent, "", envColorListName); - Globals::Instance->GetSegmentedPtrName(frameDataListAddress, parent, "", frameDataListName); + Globals::Instance->GetSegmentedPtrName(primColorListAddress, parent, "", primColorListName, + parent->workerID); + Globals::Instance->GetSegmentedPtrName(envColorListAddress, parent, "", envColorListName, + parent->workerID); + Globals::Instance->GetSegmentedPtrName(frameDataListAddress, parent, "", frameDataListName, + parent->workerID); std::string bodyStr = StringHelper::Sprintf( "\n %d, %d, %s, %s, %s,\n", animLength, colorListCount, primColorListName.c_str(), @@ -423,7 +426,8 @@ void TextureCyclingParams::DeclareReferences([[maybe_unused]] const std::string& for (const auto& tex : textureList) { - bool texFound = Globals::Instance->GetSegmentedPtrName(tex, parent, "", texName); + bool texFound = + Globals::Instance->GetSegmentedPtrName(tex, parent, "", texName, parent->workerID); // texName is a raw segmented pointer. This occurs if the texture is not declared // separately since we cannot read the format. In theory we could scan DLists for the @@ -477,9 +481,10 @@ std::string TextureCyclingParams::GetBodySourceCode() const std::string textureListName; std::string textureIndexListName; - Globals::Instance->GetSegmentedPtrName(textureListAddress, parent, "", textureListName); + Globals::Instance->GetSegmentedPtrName(textureListAddress, parent, "", textureListName, + parent->workerID); Globals::Instance->GetSegmentedPtrName(textureIndexListAddress, parent, "", - textureIndexListName); + textureIndexListName, parent->workerID); std::string bodyStr = StringHelper::Sprintf( "\n %d, %s, %s,\n", cycleLength, textureListName.c_str(), textureIndexListName.c_str()); @@ -652,7 +657,8 @@ std::string ZTextureAnimation::GetBodySourceCode() const for (const auto& entry : entries) { std::string paramName; - Globals::Instance->GetSegmentedPtrName(entry.paramsPtr, parent, "", paramName); + Globals::Instance->GetSegmentedPtrName(entry.paramsPtr, parent, "", paramName, + parent->workerID); bodyStr += StringHelper::Sprintf("\t{ %d, %d, %s },\n", entry.segment, entry.type, paramName.c_str()); diff --git a/ZAPDTR/ZAPD/ctpl_stl.h b/ZAPDTR/ZAPD/ctpl_stl.h new file mode 100644 index 000000000..8e2d9c908 --- /dev/null +++ b/ZAPDTR/ZAPD/ctpl_stl.h @@ -0,0 +1,251 @@ +/********************************************************* +* +* Copyright (C) 2014 by Vitaliy Vitsentiy +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +*********************************************************/ + + +#ifndef __ctpl_stl_thread_pool_H__ +#define __ctpl_stl_thread_pool_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + + +// thread pool to run user's functors with signature +// ret func(int id, other_params) +// where id is the index of the thread that runs the functor +// ret is some return type + + +namespace ctpl { + + namespace detail { + template + class Queue { + public: + bool push(T const & value) { + std::unique_lock lock(this->mutex); + this->q.push(value); + return true; + } + // deletes the retrieved element, do not use for non integral types + bool pop(T & v) { + std::unique_lock lock(this->mutex); + if (this->q.empty()) + return false; + v = this->q.front(); + this->q.pop(); + return true; + } + bool empty() { + std::unique_lock lock(this->mutex); + return this->q.empty(); + } + private: + std::queue q; + std::mutex mutex; + }; + } + + class thread_pool { + + public: + + thread_pool() { this->init(); } + thread_pool(int nThreads) { this->init(); this->resize(nThreads); } + + // the destructor waits for all the functions in the queue to be finished + ~thread_pool() { + this->stop(true); + } + + // get the number of running threads in the pool + int size() { return static_cast(this->threads.size()); } + + // number of idle threads + int n_idle() { return this->nWaiting; } + std::thread & get_thread(int i) { return *this->threads[i]; } + + // change the number of threads in the pool + // should be called from one thread, otherwise be careful to not interleave, also with this->stop() + // nThreads must be >= 0 + void resize(int nThreads) { + if (!this->isStop && !this->isDone) { + int oldNThreads = static_cast(this->threads.size()); + if (oldNThreads <= nThreads) { // if the number of threads is increased + this->threads.resize(nThreads); + this->flags.resize(nThreads); + + for (int i = oldNThreads; i < nThreads; ++i) { + this->flags[i] = std::make_shared>(false); + this->set_thread(i); + } + } + else { // the number of threads is decreased + for (int i = oldNThreads - 1; i >= nThreads; --i) { + *this->flags[i] = true; // this thread will finish + this->threads[i]->detach(); + } + { + // stop the detached threads that were waiting + std::unique_lock lock(this->mutex); + this->cv.notify_all(); + } + this->threads.resize(nThreads); // safe to delete because the threads are detached + this->flags.resize(nThreads); // safe to delete because the threads have copies of shared_ptr of the flags, not originals + } + } + } + + // empty the queue + void clear_queue() { + std::function * _f; + while (this->q.pop(_f)) + delete _f; // empty the queue + } + + // pops a functional wrapper to the original function + std::function pop() { + std::function * _f = nullptr; + this->q.pop(_f); + std::unique_ptr> func(_f); // at return, delete the function even if an exception occurred + std::function f; + if (_f) + f = *_f; + return f; + } + + // wait for all computing threads to finish and stop all threads + // may be called asynchronously to not pause the calling thread while waiting + // if isWait == true, all the functions in the queue are run, otherwise the queue is cleared without running the functions + void stop(bool isWait = false) { + if (!isWait) { + if (this->isStop) + return; + this->isStop = true; + for (int i = 0, n = this->size(); i < n; ++i) { + *this->flags[i] = true; // command the threads to stop + } + this->clear_queue(); // empty the queue + } + else { + if (this->isDone || this->isStop) + return; + this->isDone = true; // give the waiting threads a command to finish + } + { + std::unique_lock lock(this->mutex); + this->cv.notify_all(); // stop all waiting threads + } + for (int i = 0; i < static_cast(this->threads.size()); ++i) { // wait for the computing threads to finish + if (this->threads[i]->joinable()) + this->threads[i]->join(); + } + // if there were no threads in the pool but some functors in the queue, the functors are not deleted by the threads + // therefore delete them here + this->clear_queue(); + this->threads.clear(); + this->flags.clear(); + } + + template + auto push(F && f, Rest&&... rest) ->std::future { + auto pck = std::make_shared>( + std::bind(std::forward(f), std::placeholders::_1, std::forward(rest)...) + ); + auto _f = new std::function([pck](int id) { + (*pck)(id); + }); + this->q.push(_f); + std::unique_lock lock(this->mutex); + this->cv.notify_one(); + return pck->get_future(); + } + + // run the user's function that excepts argument int - id of the running thread. returned value is templatized + // operator returns std::future, where the user can get the result and rethrow the catched exceptins + template + auto push(F && f) ->std::future { + auto pck = std::make_shared>(std::forward(f)); + auto _f = new std::function([pck](int id) { + (*pck)(id); + }); + this->q.push(_f); + std::unique_lock lock(this->mutex); + this->cv.notify_one(); + return pck->get_future(); + } + + + private: + + // deleted + thread_pool(const thread_pool &);// = delete; + thread_pool(thread_pool &&);// = delete; + thread_pool & operator=(const thread_pool &);// = delete; + thread_pool & operator=(thread_pool &&);// = delete; + + void set_thread(int i) { + std::shared_ptr> flag(this->flags[i]); // a copy of the shared ptr to the flag + auto f = [this, i, flag/* a copy of the shared ptr to the flag */]() { + std::atomic & _flag = *flag; + std::function * _f; + bool isPop = this->q.pop(_f); + while (true) { + while (isPop) { // if there is anything in the queue + std::unique_ptr> func(_f); // at return, delete the function even if an exception occurred + (*_f)(i); + if (_flag) + return; // the thread is wanted to stop, return even if the queue is not empty yet + else + isPop = this->q.pop(_f); + } + // the queue is empty here, wait for the next command + std::unique_lock lock(this->mutex); + ++this->nWaiting; + this->cv.wait(lock, [this, &_f, &isPop, &_flag](){ isPop = this->q.pop(_f); return isPop || this->isDone || _flag; }); + --this->nWaiting; + if (!isPop) + return; // if the queue is empty and this->isDone == true or *flag then return + } + }; + this->threads[i].reset(new std::thread(f)); // compiler may not support std::make_unique() + } + + void init() { this->nWaiting = 0; this->isStop = false; this->isDone = false; } + + std::vector> threads; + std::vector>> flags; + detail::Queue *> q; + std::atomic isDone; + std::atomic isStop; + std::atomic nWaiting; // how many threads are waiting + + std::mutex mutex; + std::condition_variable cv; + }; + +} + +#endif // __ctpl_stl_thread_pool_H__ \ No newline at end of file diff --git a/ZAPDTR/lib/libgfxd/priv.h b/ZAPDTR/lib/libgfxd/priv.h index 37cb66b68..34d96f675 100644 --- a/ZAPDTR/lib/libgfxd/priv.h +++ b/ZAPDTR/lib/libgfxd/priv.h @@ -2,6 +2,8 @@ #define GFXD_PRIV_H #include "gfxd.h" +#define CONFIG_MT + #ifdef CONFIG_MT # ifdef _MSC_VER # define TLOCAL __declspec(thread) @@ -9,7 +11,7 @@ # define TLOCAL _Thread_local # endif #else -# define TLOCAL +#define TLOCAL #endif #define UCFUNC static inline