github/Shipwright
1
0
Fork 0
mirror of https://github.com/HarbourMasters/Shipwright.git synced 2025-08-21 22:03:36 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added new extract mode to ZAPD and optimized OTR gen time

Browse source
This commit is contained in:
Jack Walker 2022-03-30 18:35:16 -04:00
commit 7d94209d36
14 changed files with 239 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

22
OTRExporter/OTRExporter/DisplayListExporter.cpp
View file

@ -370,7 +370,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
//std::string fName = StringHelper::Sprintf("%s\\%s", GetParentFolderName(res).c_str(), dListDecl2->varName.c_str());
std::string fName = OTRExporter_DisplayList::GetPathToRes(res, dListDecl2->varName.c_str());
if (!File::Exists("Extract\\" + fName))
if (files.find(fName) == files.end() && !File::Exists("Extract\\" + fName))
{
MemoryStream* dlStream = new MemoryStream();
BinaryWriter dlWriter = BinaryWriter(dlStream);
@ -382,7 +382,10 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
//otrArchive->RemoveFile(fName);
#endif
File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector());
else
files[fName] = dlStream->ToVector();
//otrArchive->AddFile(fName, (uintptr_t)dlStream->ToVector().data(), dlWriter.GetBaseAddress());
}
@ -464,14 +467,17 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
//std::string fName = StringHelper::Sprintf("%s\\%s", GetParentFolderName(res).c_str(), dListDecl2->varName.c_str());
std::string fName = OTRExporter_DisplayList::GetPathToRes(res, dListDecl2->varName.c_str());
if (!File::Exists("Extract\\" + fName))
if (files.find(fName) == files.end() && !File::Exists("Extract\\" + fName))
{
MemoryStream* dlStream = new MemoryStream();
BinaryWriter dlWriter = BinaryWriter(dlStream);
Save(dList->otherDLists[i], outPath, &dlWriter);
File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector());
else
files[fName] = dlStream->ToVector();
}
}
else
@ -793,7 +799,6 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
addr -= dList->parent->baseAddress;
auto segOffset = GETSEGOFFSET(addr);
//uint32_t seg = data & 0xFFFFFFFF;
Declaration* vtxDecl = dList->parent->GetDeclarationRanged(segOffset);
//std::string vtxName = "";
//bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, dList->parent, "", vtxName);
@ -822,7 +827,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
word0 = hash >> 32;
word1 = hash & 0xFFFFFFFF;
if (!File::Exists("Extract\\" + fName))
if (files.find(fName) == files.end() && !File::Exists("Extract\\" + fName))
{
//printf("Exporting VTX Data %s\n", fName.c_str());
// Write vertices to file
@ -878,7 +883,10 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
}
}
File::WriteAllBytes("Extract\\" + fName, vtxStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, vtxStream->ToVector());
else
files[fName] = vtxStream->ToVector();
auto end = std::chrono::steady_clock::now();
size_t diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();

36
OTRExporter/OTRExporter/Main.cpp
View file

@ -25,6 +25,7 @@ std::string otrFileName = "oot.otr";
std::shared_ptr<Ship::Archive> otrArchive;
BinaryWriter* fileWriter;
std::chrono::steady_clock::time_point fileStart, resStart;
std::map<std::string, std::vector<char>> files;
void InitVersionInfo();
@ -39,6 +40,8 @@ static void ExporterParseFileMode(const std::string& buildMode, ZFileMode& fileM
{
fileMode = (ZFileMode)ExporterFileMode::BuildOTR;
printf("BOTR: Generating OTR Archive...\n");
if (File::Exists(otrFileName))
otrArchive = std::shared_ptr<Ship::Archive>(new Ship::Archive(otrFileName, true));
else
@ -54,6 +57,31 @@ static void ExporterParseFileMode(const std::string& buildMode, ZFileMode& fileM
}
}
static void ExporterProgramEnd()
{
if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory)
{
printf("Generating OTR Archive...\n");
otrArchive = Ship::Archive::CreateArchive(otrFileName, 65536 / 2);
for (auto item : files)
{
auto fileData = item.second;
otrArchive->AddFile(item.first, (uintptr_t)fileData.data(), fileData.size());
}
// Add any additional files that need to be manually copied...
auto lst = Directory::ListFiles("Extract");
for (auto item : lst)
{
auto fileData = File::ReadAllBytes(item);
otrArchive->AddFile(StringHelper::Split(item, "Extract\\")[1], (uintptr_t)fileData.data(), fileData.size());
}
}
}
static void ExporterParseArgs(int argc, char* argv[], int& i)
{
std::string arg = argv[i];
@ -85,6 +113,7 @@ static void ExporterFileBegin(ZFile* file)
static void ExporterFileEnd(ZFile* file)
{
int bp = 0;
}
static void ExporterResourceEnd(ZResource* res, BinaryWriter& writer)
@ -124,7 +153,10 @@ static void ExporterResourceEnd(ZResource* res, BinaryWriter& writer)
else
fName = StringHelper::Sprintf("%s\\%s", oName.c_str(), rName.c_str());
File::WriteAllBytes("Extract\\" + fName, strem->ToVector());
if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory)
files[fName] = strem->ToVector();
else
File::WriteAllBytes("Extract\\" + fName, strem->ToVector());
}
auto end = std::chrono::steady_clock::now();
@ -155,6 +187,8 @@ static void ImportExporters()
exporterSet->beginXMLFunc = ExporterXMLBegin;
exporterSet->endXMLFunc = ExporterXMLEnd;
exporterSet->resSaveFunc = ExporterResourceEnd;
exporterSet->endProgramFunc = ExporterProgramEnd;
exporterSet->exporters[ZResourceType::Background] = new OTRExporter_Background();
exporterSet->exporters[ZResourceType::Texture] = new OTRExporter_Texture();
exporterSet->exporters[ZResourceType::Room] = new OTRExporter_Room();

3
OTRExporter/OTRExporter/Main.h
View file

@ -2,4 +2,5 @@
#include <Archive.h>
extern std::shared_ptr<Ship::Archive> otrArchive;
extern std::shared_ptr<Ship::Archive> otrArchive;
extern std::map<std::string, std::vector<char>> files;

12
OTRExporter/OTRExporter/RoomExporter.cpp
View file

@ -452,8 +452,11 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite
BinaryWriter csWriter = BinaryWriter(csStream);
OTRExporter_Cutscene cs;
cs.Save(cmdSetCutscenes->cutscenes[0], "", &csWriter);
File::WriteAllBytes("Extract\\" + fName, csStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, csStream->ToVector());
else
files[fName] = csStream->ToVector();
//std::string fName = OTRExporter_DisplayList::GetPathToRes(res, vtxDecl->varName);
//otrArchive->AddFile(fName, (uintptr_t)csStream->ToVector().data(), csWriter.GetBaseAddress());
@ -477,7 +480,10 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite
OTRExporter_Path pathExp;
pathExp.Save(&cmdSetPathways->pathwayList, outPath, &pathWriter);
File::WriteAllBytes("Extract\\" + path, pathStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + path, pathStream->ToVector());
else
files[path] = pathStream->ToVector();
//otrArchive->AddFile(path, (uintptr_t)pathStream->ToVector().data(), pathWriter.GetBaseAddress());

9
OTRGui/src/game/game.cpp
View file

@ -76,6 +76,15 @@ void ExtractRom() {
if (result.error == NULLSTR) {
if (MoonUtils::exists("oot.otr")) MoonUtils::rm("oot.otr");
if (MoonUtils::exists("Extract")) MoonUtils::rm("Extract");
MoonUtils::mkdir("Extract");
MoonUtils::copy("tmp/baserom/Audiobank", "Extract/Audiobank");
MoonUtils::copy("tmp/baserom/Audioseq", "Extract/Audioseq");
MoonUtils::copy("tmp/baserom/Audiotable", "Extract/Audiotable");
MoonUtils::copy("tmp/baserom/version", "Extract/version");
MoonUtils::copy("assets/game/", "Extract/assets/");
startWorker(version);
extracting = true;
}

84
OTRGui/src/impl/extractor/extractor.cpp
View file

@ -14,6 +14,7 @@
#endif
namespace Util = MoonUtils;
static bool oldExtractMode = false;
static int maxResources = 0;
static int extractedResources = 0;
bool buildingOtr = false;
@ -36,19 +37,16 @@ std::string GetXMLVersion(RomVersion version)
}
void BuildOTR(const std::string output) {
Util::copy("tmp/baserom/Audiobank", "Extract/Audiobank");
Util::copy("tmp/baserom/Audioseq", "Extract/Audioseq");
Util::copy("tmp/baserom/Audiotable", "Extract/Audiotable");
Util::copy("tmp/baserom/version", "Extract/version");
Util::copy("assets/game/", "Extract/assets/");
std::string execStr = Util::format("assets/extractor/%s", isWindows() ? "ZAPD.exe" : "ZAPD.out") + " botr -se OTR";
ProcessResult result = NativeFS->LaunchProcess(execStr);
if(result.exitCode != 0) {
std::cout << "\nError when building the OTR file with error code: " << result.exitCode << " !" << std::endl;
std::cout << "Aborting...\n" << std::endl;
if (oldExtractMode)
{
std::string execStr = Util::format("assets/extractor/%s", isWindows() ? "ZAPD.exe" : "ZAPD.out") + " botr -se OTR";
ProcessResult result = NativeFS->LaunchProcess(execStr);
if (result.exitCode != 0) {
std::cout << "\nError when building the OTR file with error code: " << result.exitCode << " !" << std::endl;
std::cout << "Aborting...\n" << std::endl;
}
}
setCurrentStep("Done!");
if (output == ".") return;
@ -85,38 +83,62 @@ void startWorker(RomVersion version) {
Util::write("tmp/baserom/version", (char*)&version.crc, sizeof(version.crc));
std::vector<std::string> files;
Util::dirscan(path, files);
std::vector<std::string> xmlFiles;
const int num_threads = std::thread::hardware_concurrency();
ctpl::thread_pool pool(num_threads / 2);
for(auto &file : files) {
if (file.find(".xml") != std::string::npos) xmlFiles.push_back(file);
}
if (oldExtractMode)
{
std::vector<std::string> files;
Util::dirscan(path, files);
std::vector<std::string> xmlFiles;
for (auto& file : xmlFiles) {
if(single_thread) {
ExtractFunc(file, version);
} else {
pool.push([file, version](int) {
ExtractFunc(file, version);
});
const int num_threads = std::thread::hardware_concurrency();
ctpl::thread_pool pool(num_threads / 2);
for (auto& file : files) {
if (file.find(".xml") != std::string::npos) xmlFiles.push_back(file);
}
}
maxResources = xmlFiles.size();
for (auto& file : xmlFiles) {
if (single_thread) {
ExtractFunc(file, version);
}
else {
pool.push([file, version](int) {
ExtractFunc(file, version);
});
}
}
maxResources = xmlFiles.size();
}
else
{
std::string execStr = Util::format("assets/extractor/%s", isWindows() ? "ZAPD.exe" : "ZAPD.out");
std::string args = Util::format(" ed -eh -i %s -b tmp/baserom/ -o %s -osf %s -gsf 1 -rconf assets/extractor/Config_%s.xml -se OTR %s", path.c_str(), path + "../", path + "../", GetXMLVersion(version).c_str(), "");
ProcessResult result = NativeFS->LaunchProcess(execStr + args);
if (result.exitCode != 0) {
std::cout << "\nError when extracting the ROM with error code: " << result.exitCode << " !" << std::endl;
std::cout << "Aborting...\n" << std::endl;
}
else
{
printf("All done?\n");
}
maxResources = 1;
}
}
void updateWorker(const std::string& output) {
if (maxResources > 0 && !buildingOtr && extractedResources >= maxResources) {
if (maxResources > 0 && !buildingOtr && (extractedResources >= maxResources || !oldExtractMode))
{
setCurrentStep("Building OTR...");
if (skipFrames < 3) {
skipFrames++;
return;
}
buildingOtr = true;
if (single_thread){
if (single_thread || !oldExtractMode){
BuildOTR(output);
return;
}

1
ZAPDTR/ZAPD/Globals.h
View file

@ -36,6 +36,7 @@ public:
ExporterSetFuncVoid3 beginXMLFunc = nullptr;
ExporterSetFuncVoid3 endXMLFunc = nullptr;
ExporterSetResSave resSaveFunc = nullptr;
ExporterSetFuncVoid3 endProgramFunc = nullptr;
};
class Globals

107
ZAPDTR/ZAPD/Main.cpp
View file

@ -240,6 +240,8 @@ int main(int argc, char* argv[])
fileMode = ZFileMode::BuildBlob;
else if (buildMode == "e")
fileMode = ZFileMode::Extract;
else if (buildMode == "ed")
fileMode = ZFileMode::ExtractDirectory;
else if (exporterSet != nullptr && exporterSet->parseFileModeFunc != nullptr)
exporterSet->parseFileModeFunc(buildMode, fileMode);
@ -249,6 +251,8 @@ int main(int argc, char* argv[])
return 1;
}
Globals::Instance->fileMode = fileMode;
// We've parsed through our commands once. If an exporter exists, it's been set by now.
// Now we'll parse through them again but pass them on to our exporter if one is available.
@ -267,39 +271,107 @@ int main(int argc, char* argv[])
}
// TODO: switch
if (fileMode == ZFileMode::Extract || fileMode == ZFileMode::BuildSourceFile)
if (fileMode == ZFileMode::Extract || fileMode == ZFileMode::BuildSourceFile || fileMode == ZFileMode::ExtractDirectory)
{
bool procFileModeSuccess = false;
if (exporterSet != nullptr && exporterSet->processFileModeFunc != nullptr)
procFileModeSuccess = exporterSet->processFileModeFunc(fileMode);
if (!procFileModeSuccess)
{
bool parseSuccessful;
for (auto& extFile : Globals::Instance->cfg.externalFiles)
if (fileMode == ZFileMode::ExtractDirectory)
{
fs::path externalXmlFilePath =
Globals::Instance->cfg.externalXmlFolder / extFile.xmlPath;
std::vector<std::string> fileList =
Directory::ListFiles(Globals::Instance->inputPath.string());
if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO)
bool parseSuccessful;
auto start = std::chrono::steady_clock::now();
for (int i = 0; i < fileList.size(); i++)
{
printf("Parsing external file from config: '%s'\n",
externalXmlFilePath.c_str());
printf("(%i / %i): %s\n", (i+1), fileList.size(), fileList[i].c_str());
for (auto& extFile : Globals::Instance->cfg.externalFiles)
{
fs::path externalXmlFilePath =
Globals::Instance->cfg.externalXmlFolder / extFile.xmlPath;
if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO)
{
printf("Parsing external file from config: '%s'\n",
externalXmlFilePath.c_str());
}
parseSuccessful = Parse(externalXmlFilePath, Globals::Instance->baseRomPath,
extFile.outPath, ZFileMode::ExternalFile);
if (!parseSuccessful)
return 1;
}
parseSuccessful = Parse(fileList[i], Globals::Instance->baseRomPath,
Globals::Instance->outputPath, fileMode);
if (!parseSuccessful)
return 1;
/*for (auto file : Globals::Instance->files)
delete file;*/
for (int i = Globals::Instance->cfg.externalFiles.size(); i < Globals::Instance->files.size(); i++)
{
//if (!Globals::Instance->files[i]->isExternalFile)
{
delete Globals::Instance->files[i];
Globals::Instance->files.erase(Globals::Instance->files.begin() + i);
i--;
}
}
//Globals::Instance->files.clear();
Globals::Instance->externalFiles.clear();
Globals::Instance->segments.clear();
Globals::Instance->cfg.segmentRefFiles.clear();
}
parseSuccessful = Parse(externalXmlFilePath, Globals::Instance->baseRomPath,
extFile.outPath, ZFileMode::ExternalFile);
auto end = std::chrono::steady_clock::now();
auto diff =
std::chrono::duration_cast<std::chrono::seconds>(end - start).count();
printf("Generated OTR File Data in %i seconds\n", diff);
}
else
{
bool parseSuccessful;
for (auto& extFile : Globals::Instance->cfg.externalFiles)
{
fs::path externalXmlFilePath =
Globals::Instance->cfg.externalXmlFolder / extFile.xmlPath;
if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO)
{
printf("Parsing external file from config: '%s'\n",
externalXmlFilePath.c_str());
}
parseSuccessful = Parse(externalXmlFilePath, Globals::Instance->baseRomPath,
extFile.outPath, ZFileMode::ExternalFile);
if (!parseSuccessful)
return 1;
}
parseSuccessful =
Parse(Globals::Instance->inputPath, Globals::Instance->baseRomPath,
Globals::Instance->outputPath, fileMode);
if (!parseSuccessful)
return 1;
}
parseSuccessful = Parse(Globals::Instance->inputPath, Globals::Instance->baseRomPath,
Globals::Instance->outputPath, fileMode);
if (!parseSuccessful)
return 1;
}
}
else if (fileMode == ZFileMode::BuildTexture)
@ -326,6 +398,9 @@ int main(int argc, char* argv[])
overlay->GetSourceOutputCode(""));
}
if (exporterSet != nullptr && exporterSet->endProgramFunc != nullptr)
exporterSet->endProgramFunc();
delete g;
return 0;
}

7
ZAPDTR/ZAPD/ZBackground.cpp
View file

@ -150,8 +150,11 @@ std::string ZBackground::GetExternalExtension() const
void ZBackground::Save(const fs::path& outFolder)
{
fs::path filepath = outFolder / (outName + "." + GetExternalExtension());
File::WriteAllBytes(filepath.string(), data);
if (!Globals::Instance->otrMode)
{
fs::path filepath = outFolder / (outName + "." + GetExternalExtension());
File::WriteAllBytes(filepath.string(), data);
}
}
std::string ZBackground::GetBodySourceCode() const

10
ZAPDTR/ZAPD/ZFile.cpp
View file

@ -181,7 +181,7 @@ void ZFile::ParseXML(tinyxml2::XMLElement* reader, const std::string& filename)
}
}
if (mode == ZFileMode::Extract || mode == ZFileMode::ExternalFile)
if (mode == ZFileMode::Extract || mode == ZFileMode::ExternalFile || mode == ZFileMode::ExtractDirectory)
{
if (!File::Exists((basePath / name).string()))
{
@ -260,7 +260,7 @@ void ZFile::ParseXML(tinyxml2::XMLElement* reader, const std::string& filename)
{
ZResource* nRes = nodeMap[nodeName](this);
if (mode == ZFileMode::Extract || mode == ZFileMode::ExternalFile)
if (mode == ZFileMode::Extract || mode == ZFileMode::ExternalFile || mode == ZFileMode::ExtractDirectory)
nRes->ExtractFromXML(child, rawDataIndex);
switch (nRes->GetResourceType())
@ -813,7 +813,8 @@ void ZFile::GenerateSourceHeaderFiles()
if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO)
printf("Writing H file: %s\n", headerFilename.c_str());
File::WriteAllText(headerFilename, formatter.GetOutput());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllText(headerFilename, formatter.GetOutput());
}
std::string ZFile::GetHeaderInclude() const
@ -1190,6 +1191,9 @@ void ZFile::HandleUnaccountedData()
uint32_t lastSize = 0;
std::vector<offset_t> declsAddresses;
if (Globals::Instance->otrMode)
return;
for (const auto& item : declarations)
{
declsAddresses.push_back(item.first);

1
ZAPDTR/ZAPD/ZFile.h
View file

@ -16,6 +16,7 @@ enum class ZFileMode
BuildBackground,
Extract,
ExternalFile,
ExtractDirectory,
Invalid,
Custom = 1000, // Used for exporter file modes
};

3
ZAPDTR/ZAPD/ZLimb.cpp
View file

@ -218,6 +218,9 @@ size_t ZLimb::GetRawDataSize() const
std::string ZLimb::GetBodySourceCode() const
{
if (Globals::Instance->otrMode)
return "";
std::string dListStr;
std::string dListStr2;
Globals::Instance->GetSegmentedArrayIndexedName(dListPtr, 8, parent, "Gfx", dListStr);

4
ZAPDTR/ZAPD/ZPlayerAnimationData.cpp
View file

@ -3,6 +3,7 @@
#include "Utils/BitConverter.h"
#include "Utils/StringHelper.h"
#include "ZFile.h"
#include <Globals.h>
REGISTER_ZFILENODE(PlayerAnimationData, ZPlayerAnimationData);
@ -54,6 +55,9 @@ std::string ZPlayerAnimationData::GetBodySourceCode() const
{
std::string declaration = "";
if (Globals::Instance->otrMode)
return "";
size_t index = 0;
for (const auto& entry : limbRotData)
{

4
soh/src/overlays/actors/ovl_Elf_Msg/z_elf_msg.c
View file

@ -138,6 +138,10 @@ void ElfMsg_CallNaviCylinder(ElfMsg* this, GlobalContext* globalCtx) {
Player* player = GET_PLAYER(globalCtx);
EnElf* navi = (EnElf*)player->naviActor;
// This fixes a crash when using a grotto exit when you never properly entered
if (navi == NULL)
return;
if (ElfMsg_WithinXZDistance(&player->actor.world.pos, &this->actor.world.pos, this->actor.scale.x * 100.0f) &&
(this->actor.world.pos.y <= player->actor.world.pos.y) &&
((player->actor.world.pos.y - this->actor.world.pos.y) < (100.0f * this->actor.scale.y))) {