github/Shipwright
1
0
Fork 0
mirror of https://github.com/HarbourMasters/Shipwright.git synced 2025-08-22 14:23:44 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

cleanup extraction scripts

Browse source
This commit is contained in:
Random06457 2022-05-03 16:26:30 +09:00
commit 9ef1223330
9 changed files with 186 additions and 3452 deletions
Download patch file Download diff file Expand all files Collapse all files

4
BUILDING.md
View file

@ -23,8 +23,8 @@
# Clone the repo
git clone git@github.com:HarbourMasters/ShipWright.git
cd ShipWright
# Copy the baserom to the soh folder
cp .../baserom_non_mq.z64 soh
# Copy the baserom to the OTRExporter folder
cp <path to your ROM> OTRExporter
# Build the docker image
sudo docker build . -t soh
# Run the docker image with the working directory mounted to /soh

129
OTRExporter/extract_assets.py
View file

@ -1,45 +1,9 @@
#!/usr/bin/env python3
# How to use:
# Place a rom in this directory then run the script.
# If you are using multiple roms, the script will let you choose one.
# To choose with a commandline argument:
# Python3 extract_assets.py <number>
# Invalid input results in the first rom being selected
import json, os, signal, time, sys, shutil, glob
from multiprocessing import Pool, cpu_count, Event, Manager, ProcessError
from enum import Enum
import os, sys, shutil
import shutil
romVer = "..\\soh\\baserom_non_mq.z64"
roms = [];
checksums = ["", "", ""];
class Checksums(Enum):
OOT_NTSC_10 = "EC7011B7"
OOT_NTSC_11 = "D43DA81F"
OOT_NTSC_12 = "693BA2AE"
OOT_PAL_10 = "B044B569"
OOT_PAL_11 = "B2055FBD"
OOT_NTSC_JP_GC_CE = "F7F52DB8"
OOT_NTSC_JP_GC = "F611F4BA"
OOT_NTSC_US_GC = "F3DD35BA"
OOT_PAL_GC = "09465AC3"
OOT_NTSC_JP_MQ = "F43B45BA"
OOT_NTSC_US_MQ = "F034001A"
OOT_PAL_MQ = "1D4136F3"
OOT_PAL_GC_DBG1 = "871E1C92"
OOT_PAL_GC_DBG2 = "87121EFE"
OOT_PAL_GC_MQ_DBG = "917D18F6"
OOT_IQUE_TW = "3D81FB3E"
OOT_IQUE_CN = "B1E1E07B"
OOT_UNKNOWN = "FFFFFFFF"
CompatibleChecksums = [
Checksums.OOT_PAL_GC,
Checksums.OOT_PAL_GC_DBG1
]
from rom_info import Z64Rom
import rom_chooser
def BuildOTR(xmlPath, rom):
shutil.copytree("assets", "Extract/assets")
@ -55,95 +19,14 @@ def BuildOTR(xmlPath, rom):
print("Aborting...", file=os.sys.stderr)
print("\n")
def checkChecksum(rom):
r = open(rom, "rb")
r.seek(16)
bytes = r.read(4).hex().upper()
r.close()
for checksum in Checksums:
if (checksum.value == bytes):
for compat in CompatibleChecksums:
if (checksum.name == compat.name):
print("Compatible rom found!")
return checksum
print("Valid oot rom found. However, not compatible with SoH.")
print("Compatible roms:")
for compat in CompatibleChecksums:
print(compat.name+" | 0x"+compat.value)
sys.exit(1)
print("Wrong rom! No valid checksum found")
sys.exit(1)
def main():
romToUse = "";
for file in glob.glob("*.z64"):
roms.append(file)
if not (roms):
print("Error: No roms located, place one in the OTRExporter directory", file=os.sys.stderr)
sys.exit(1)
if (len(roms) > 1):
# If commandline args exist
if (len(sys.argv) > 1):
try:
if ((int(sys.argv[1]) - 1) < 1):
romToUse = roms[0]
elif ((int(sys.argv[1]) - 1) > len(roms)):
romToUse = roms[len(roms) - 1]
else:
romToUse = roms[int(sys.argv[1]) - 1]
except:
romToUse = roms[0]
# No commandline args, select rom using user input
else:
print(str(len(roms))+" roms found, please select one by pressing 1-"+str(len(roms)))
count = 1
for list in range(len(roms)):
print(str(count)+". "+roms[list])
count += 1
while(1):
try:
selection = int(input())
except:
print("Bad input. Try again with the number keys.")
continue
if (selection < 1 or selection > len(roms)):
print("Bad input. Try again.")
continue
else: break
romToUse = roms[selection - 1]
else:
romToUse = roms[0]
match checkChecksum(romToUse):
case Checksums.OOT_PAL_GC:
xmlVer = "GC_NMQ_PAL_F"
case Checksums.OOT_PAL_GC_DBG1:
xmlVer = "GC_NMQ_D"
case _: # default case
xmlVer = "GC_MQ_D"
rom_path = rom_chooser.chooseROM()
rom = Z64Rom(rom_path)
if (os.path.exists("Extract")):
shutil.rmtree("Extract")
BuildOTR("../soh/assets/xml/" + xmlVer + "/", romToUse)
BuildOTR("../soh/assets/xml/" + rom.version.xml_ver + "/", rom_path)
if __name__ == "__main__":
main()

125
OTRExporter/extract_assets_old.py
View file

@ -1,125 +0,0 @@
#!/usr/bin/env python3
import argparse, json, os, signal, time, sys, shutil
from multiprocessing import Pool, cpu_count, Event, Manager, ProcessError
import shutil
def SignalHandler(sig, frame):
print(f'Signal {sig} received. Aborting...')
mainAbort.set()
# Don't exit immediately to update the extracted assets file.
def BuildOTR():
shutil.copyfile("baserom/Audiobank", "Extract/Audiobank")
shutil.copyfile("baserom/Audioseq", "Extract/Audioseq")
shutil.copyfile("baserom/Audiotable", "Extract/Audiotable")
shutil.copytree("assets", "Extract/assets")
execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out"
execStr += " botr -se OTR"
print(execStr)
exitValue = os.system(execStr)
if exitValue != 0:
print("\n")
print("Error when building the OTR file...", file=os.sys.stderr)
print("Aborting...", file=os.sys.stderr)
print("\n")
def ExtractFile(xmlPath, outputPath, outputSourcePath):
execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out"
execStr += " e -eh -i %s -b baserom/ -o %s -osf %s -gsf 1 -rconf CFG/Config.xml -se OTR" % (xmlPath, outputPath, outputSourcePath)
if "overlays" in xmlPath:
execStr += " --static"
print(execStr)
exitValue = os.system(execStr)
#exitValue = 0
if exitValue != 0:
print("\n")
print("Error when extracting from file " + xmlPath, file=os.sys.stderr)
print("Aborting...", file=os.sys.stderr)
print("\n")
def ExtractFunc(fullPath):
*pathList, xmlName = fullPath.split(os.sep)
objectName = os.path.splitext(xmlName)[0]
outPath = os.path.join("..\\soh\\assets\\", *pathList[5:], objectName)
os.makedirs(outPath, exist_ok=True)
outSourcePath = outPath
ExtractFile(fullPath, outPath, outSourcePath)
def initializeWorker(abort, test):
global globalAbort
globalAbort = abort
def main():
parser = argparse.ArgumentParser(description="baserom asset extractor")
parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep")
parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true")
parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true")
parser.add_argument("-v", "--version", help="Sets game version.")
args = parser.parse_args()
global mainAbort
mainAbort = Event()
manager = Manager()
signal.signal(signal.SIGINT, SignalHandler)
extractedAssetsTracker = manager.dict()
xmlVer = "GC_NMQ_D"
if (args.version == "gc_pal_nmpq"):
xmlVer = "GC_NMQ_PAL_F"
elif (args.version == "dbg_mq"):
xmlVer = "GC_MQ_D"
asset_path = args.single
if asset_path is not None:
fullPath = os.path.join("..\\soh\\assets", "xml", asset_path + ".xml")
if not os.path.exists(fullPath):
print(f"Error. File {fullPath} doesn't exists.", file=os.sys.stderr)
exit(1)
ExtractFunc(fullPath)
else:
extract_text_path = "assets/text/message_data.h"
if os.path.isfile(extract_text_path):
extract_text_path = None
extract_staff_text_path = "assets/text/message_data_staff.h"
if os.path.isfile(extract_staff_text_path):
extract_staff_text_path = None
xmlFiles = []
for currentPath, _, files in os.walk(os.path.join("..\\soh\\assets\\xml\\", xmlVer)):
for file in files:
fullPath = os.path.join(currentPath, file)
if file.endswith(".xml"):
xmlFiles.append(fullPath)
try:
numCores = 2
print("Extracting assets with " + str(numCores) + " CPU cores.")
with Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, 0)) as p:
p.map(ExtractFunc, xmlFiles)
except Exception as e:
print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr)
print("Disabling mutliprocessing.", file=os.sys.stderr)
initializeWorker(mainAbort, 0)
for singlePath in xmlFiles:
ExtractFunc(singlePath)
BuildOTR()
shutil.rmtree("Extract")
if __name__ == "__main__":
main()

53
OTRExporter/extract_baserom.py Normal file
View file

@ -0,0 +1,53 @@
#!/usr/bin/python3
import os
import sys
import struct
from multiprocessing import Pool, cpu_count
from rom_info import Z64Rom
import rom_chooser
rom = None
def initialize_worker(input_rom):
global rom
rom = input_rom
def ExtractFunc(i):
dma_file = rom.getDmaEntryByIndex(i)
dma_data = rom.readDmaEntry(dma_file)
filename = '../soh/baserom/' + rom.version.file_table[i]
print('extracting ' + filename + " (0x%08X, 0x%08X)" % (dma_file.virtStart, dma_file.virtEnd))
try:
with open(filename, 'wb') as f:
f.write(dma_data)
except IOError:
print('failed to write file ' + filename)
# TODO: handle this better
if dma_file.compressed:
os.system('tools/yaz0 -d ' + filename + ' ' + filename)
#####################################################################
def main():
try:
os.mkdir('../soh/baserom')
except:
pass
rom_path = rom_chooser.chooseROM()
input_rom = Z64Rom(rom_path)
# extract files
num_cores = cpu_count()
print("Extracting baserom with " + str(num_cores) + " CPU cores.")
with Pool(num_cores, initialize_worker, (input_rom,)) as p:
p.map(ExtractFunc, range(len(input_rom.version.file_table)))
if __name__ == "__main__":
main()

1608
OTRExporter/extract_baserom_debug.py
View file

File diff suppressed because it is too large Load diff

1586
OTRExporter/extract_baserom_gc.py
View file

File diff suppressed because it is too large Load diff

37
OTRExporter/rom_chooser.py Normal file
View file

@ -0,0 +1,37 @@
import os, sys, glob
from rom_info import Z64Rom
def chooseROM():
roms = []
for file in glob.glob("*.z64"):
if Z64Rom.isValidRom(file):
roms.append(file)
if not (roms):
print("Error: No roms located, place one in the OTRExporter directory", file=os.sys.stderr)
sys.exit(1)
if (len(roms) == 1):
return roms[0]
print(str(len(roms))+ " roms found, please select one by pressing 1-"+str(len(roms)))
for i in range(len(roms)):
print(str(i+1)+ ". " + roms[i])
while(1):
try:
selection = int(input())
except:
print("Bad input. Try again with the number keys.")
continue
if (selection < 1 or selection > len(roms)):
print("Bad input. Try again.")
continue
else: break
return roms[selection - 1]

87
OTRExporter/rom_info.py Normal file
View file

@ -0,0 +1,87 @@
from enum import Enum
from tabnanny import check
import struct
class Checksums(Enum):
OOT_NTSC_10 = "EC7011B7"
OOT_NTSC_11 = "D43DA81F"
OOT_NTSC_12 = "693BA2AE"
OOT_PAL_10 = "B044B569"
OOT_PAL_11 = "B2055FBD"
OOT_NTSC_JP_GC_CE = "F7F52DB8"
OOT_NTSC_JP_GC = "F611F4BA"
OOT_NTSC_US_GC = "F3DD35BA"
OOT_PAL_GC = "09465AC3"
OOT_NTSC_JP_MQ = "F43B45BA"
OOT_NTSC_US_MQ = "F034001A"
OOT_PAL_MQ = "1D4136F3"
OOT_PAL_GC_DBG1 = "871E1C92"
OOT_PAL_GC_DBG2 = "87121EFE"
OOT_PAL_GC_MQ_DBG = "917D18F6"
OOT_IQUE_TW = "3D81FB3E"
OOT_IQUE_CN = "B1E1E07B"
OOT_UNKNOWN = "FFFFFFFF"
@classmethod
def has_value(self, value):
return value in self._value2member_map_
class RomVersion:
def __init__(self, file_table_path, file_table_off, xml_ver):
self.file_table_off = file_table_off
self.xml_ver = xml_ver
with open(file_table_path, 'r') as f:
self.file_table = [line.strip('\n') for line in f]
ROM_INFO_TABLE = dict()
ROM_INFO_TABLE[Checksums.OOT_PAL_GC] = RomVersion("CFG/filelists/gamecube_pal.txt", 0x7170, "GC_NMQ_PAL_F")
ROM_INFO_TABLE[Checksums.OOT_PAL_GC_DBG1] = RomVersion("CFG/filelists/dbg.txt", 0x12F70, "GC_NMQ_D")
class RomDmaEntry:
def __init__(self, rom, i):
off = rom.version.file_table_off + 16 * i
(self.virtStart, \
self.virtEnd, \
self.physStart, \
self.physEnd) = struct.unpack('>IIII', rom.rom_data[off:off+4*4])
self.compressed = self.physEnd != 0
self.size = self.physEnd - self.physStart \
if self.compressed \
else self.virtEnd - self.virtStart
self.name = rom.version.file_table[i]
class Z64Rom:
def __init__(self, file_path):
self.file_path = file_path
with open(file_path, 'rb') as f:
self.rom_data = f.read()
self.is_valid = len(self.rom_data) > 20 * 1024 * 1024
if not self.is_valid:
return
# get checkum
checksum_str = self.rom_data[16:16+4].hex().upper()
self.checksum = Checksums(checksum_str) if Checksums.has_value(checksum_str) else Checksums.OOT_UNKNOWN
if self.checksum == Checksums.OOT_UNKNOWN:
self.is_valid = False
return
# get rom version
self.version = ROM_INFO_TABLE[self.checksum]
def getDmaEntryByIndex(self, i):
return RomDmaEntry(self, i)
def readDmaEntry(self, entry):
return self.rom_data[entry.physStart:entry.physStart + entry.size]
@staticmethod
def isValidRom(rom_path):
return Z64Rom(rom_path).is_valid

9
soh/Makefile
View file

@ -8,13 +8,6 @@ ZAPD := ../ZAPDTR/ZAPD.out
LIBULTRASHIP := ../libultraship/libultraship.a
ZAPDUTILS := ../ZAPDTR/ZAPDUtils/ZAPDUtils.a
ROM_DEBUG ?= 0
EXTRACT_BASEROM := ../OTRExporter/extract_baserom_gc.py
ifneq ($(ROM_DEBUG), 0)
EXTRACT_BASEROM := ../OTRExporter/extract_baserom_debug.py
endif
ASAN ?= 0
DEBUG ?= 1
OPTFLAGS ?= -O0
@ -127,7 +120,7 @@ all:
$(MAKE) $(TARGET)
setup:
python3 $(EXTRACT_BASEROM)
cd ../OTRExporter && python3 extract_baserom.py
$(MAKE) mpq
mpq: