Merge branch 'actor-accessibility-experiments' of https://www.github.com/caturria/shipwright into actor-accessibility-experiments

This commit is contained in:
Ryan 2023-08-30 15:06:41 -04:00
commit 08a4755477
13 changed files with 174 additions and 50 deletions

View file

@ -296,7 +296,10 @@ void accessible_goma(AccessibleActor* actor) {
}
void accessible_door_of_time(AccessibleActor* actor) {
ActorAccessibility_PlaySoundForActor(actor, 0, NA_SE_EV_DIAMOND_SWITCH, false);
ActorAccessibility_PlaySampleForActor(actor, 0, "Chanting", false);
ActorAccessibility_SetSoundPitch(actor, 0, 1.0);
//ActorAccessibility_PlaySoundForActor(actor, 0, NA_SE_EV_DIAMOND_SWITCH, false);
}
void accessible_sticks(AccessibleActor* actor) {

View file

@ -16,6 +16,12 @@
#include <unordered_set>
#include "soh/Enhancements/speechsynthesizer/SpeechSynthesizer.h"
#include "soh/Enhancements/tts/tts.h"
#include "soh/Enhancements/game-interactor/GameInteractor.h"
#include "soh/Enhancements/audio/AudioDecoder.h"
extern "C" {
extern PlayState* gPlayState;
extern bool freezeGame;
}
const char* GetLanguageCode();
@ -43,14 +49,16 @@ typedef std::map<s32, VAList_t> VAZones_t;//Maps room/ scene indices to their co
typedef std::unordered_set<s16> SceneList_t;//A list of scenes which have already been visited (since the game was launched). Used to prevent re-creation of terrain VAs every time the player reloads a scene.
typedef struct {
std::string hexName;
std::string path;
std::shared_ptr<LUS::File> resource;
std::shared_ptr<s16*> decodedSample;//Set if the record is for a raw sample as opposed to a SFX.
}SfxRecord;
class ActorAccessibility {
public:
bool isOn = false;
uint64_t nextActorID;
int isOn = 0;
uint64_t nextActorID = 0;
SupportedActors_t supportedActors;
TrackedActors_t trackedActors;
AccessibleActorList_t accessibleActorList;
@ -59,7 +67,8 @@ class ActorAccessibility {
AccessibleAudioEngine* audioEngine;
SfxExtractor sfxExtractor;
std::unordered_map<s16, SfxRecord> sfxMap;//Maps internal sfx to external (prerendered) resources.
std::unordered_map<std::string, SfxRecord> sampleMap;//Similar to above, but this one maps raw audio samples as opposed to SFX.
int extractSfx = 0;
};
static ActorAccessibility* aa;
@ -68,12 +77,48 @@ uint64_t ActorAccessibility_GetNextID() {
aa->nextActorID++;
return result;
}
void ActorAccessibility_Init() {
aa = new ActorAccessibility();
ActorAccessibility_InitAudio();
ActorAccessibility_InitActors();
// Hooks for game-interactor.
void ActorAccessibility_OnActorInit(void* actor) {
ActorAccessibility_TrackNewActor((Actor*)actor);
}
void ActorAccessibility_OnGameFrameUpdate() {
if (gPlayState == NULL)
return;
ActorAccessibility_RunAccessibilityForAllActors(gPlayState);
}
void ActorAccessibility_OnActorDestroy(void* actor)
{
ActorAccessibility_RemoveTrackedActor((Actor*) actor);
}
void ActorAccessibility_OnGameStillFrozen()
{
if (gPlayState == NULL)
return;
if (aa->extractSfx)
ActorAccessibility_HandleSoundExtractionMode(gPlayState);
}
void ActorAccessibility_Init() {
aa = new ActorAccessibility();
aa->isOn = CVarGetInteger("gA11yAudioInteraction", 0);
if (!aa->isOn)
return;
aa->extractSfx = CVarGetInteger("gExtractSfx", 0);
if (aa->extractSfx)
freezeGame = true;
ActorAccessibility_InitAudio();
ActorAccessibility_InitActors();
GameInteractor::Instance->RegisterGameHook<GameInteractor::OnActorInit>(ActorAccessibility_OnActorInit);
GameInteractor::Instance->RegisterGameHook<GameInteractor::OnActorDestroy>(ActorAccessibility_OnActorDestroy);
GameInteractor::Instance->RegisterGameHook<GameInteractor::OnGameFrameUpdate>(ActorAccessibility_OnGameFrameUpdate);
GameInteractor::Instance->RegisterGameHook<GameInteractor::OnGameStillFrozen>(ActorAccessibility_OnGameStillFrozen);
}
void ActorAccessibility_Shutdown() {
ActorAccessibility_ShutdownAudio();
delete aa;
@ -88,7 +133,7 @@ void ActorAccessibility_Shutdown() {
policy->pitch = 1.5;
policy->runsAlways = false;
policy->sound = sfx;
policy->volume = 1.0;
policy->volume = 0.5;
policy->initUserData = NULL;
policy->cleanupUserData = NULL;
policy->pitchModifier = 0.1;
@ -114,7 +159,7 @@ int ActorAccessibility_GetRandomStartingFrameCount(int min, int max) {
}
void ActorAccessibility_TrackNewActor(Actor* actor) {
void ActorAccessibility_TrackNewActor(Actor * actor) {
// Don't track actors for which no accessibility policy has been configured.
ActorAccessibilityPolicy* policy = ActorAccessibility_GetPolicyForActor(actor->id);
if (policy == NULL)
@ -178,12 +223,23 @@ void ActorAccessibility_TrackNewActor(Actor* actor) {
Audio_PlaySoundGeneral(sfxId, &actor->projectedPos, 4, &actor->currentPitch, &actor->currentVolume, &actor->currentReverb);
}
const char* ActorAccessibility_MapSfxToExternalAudio(s16 sfxId);
void ActorAccessibility_PlaySound(void* handle, int slot, s16 sfxId, bool looping)
{
const char* path = ActorAccessibility_MapSfxToExternalAudio(sfxId);
if (path == NULL)
return;
aa->audioEngine->playSound((uintptr_t)handle, slot, path, looping);
}
const char* ActorAccessibility_MapRawSampleToExternalAudio(const char* name);
void ActorAccessibility_PlayRawSample(void* handle, int slot, const char* name, bool looping)
{
const char* path = ActorAccessibility_MapRawSampleToExternalAudio(name);
if (path == NULL)
return;
aa->audioEngine->playSound((uintptr_t)handle, slot, path, looping);
}
void ActorAccessibility_StopSound(void* handle, int slot)
{
aa->audioEngine->stopSound((uintptr_t) handle, slot);
@ -231,17 +287,29 @@ void ActorAccessibility_TrackNewActor(Actor* actor) {
aa->audioEngine->seekSound((uintptr_t)handle, slot, offset);
}
void ActorAccessibility_ConfigureSoundForActor(AccessibleActor* actor, int slot)
{
ActorAccessibility_SetSoundPitch(actor, slot, actor->policy.pitch);
ActorAccessibility_SetPitchBehindModifier(actor, slot, actor->policy.pitchModifier);
ActorAccessibility_SetSoundPos(actor, slot, &actor->projectedPos, actor->xyzDistToPlayer,
actor->policy.distance);
ActorAccessibility_SetSoundVolume(actor, slot, actor->policy.volume);
actor->managedSoundSlots[slot] = true;
}
void ActorAccessibility_PlaySoundForActor(AccessibleActor* actor, int slot, s16 sfxId, bool looping)
{
if (slot < 0 || slot > NUM_MANAGED_SOUND_SLOTS)
return;
ActorAccessibility_PlaySound(actor, slot, sfxId, looping);
ActorAccessibility_SetSoundPitch(actor, slot, actor->policy.pitch);
ActorAccessibility_SetPitchBehindModifier(actor, slot, actor->policy.pitchModifier);
ActorAccessibility_SetSoundPos(actor, slot, &actor->projectedPos, actor->xzDistToPlayer,
actor->policy.distance);
ActorAccessibility_SetSoundVolume(actor, slot, actor->policy.volume);
actor->managedSoundSlots[slot] = true;
ActorAccessibility_ConfigureSoundForActor(actor, slot);
}
void ActorAccessibility_PlaySampleForActor(AccessibleActor* actor, int slot, const char* name, bool looping)
{
if (slot < 0 || slot > NUM_MANAGED_SOUND_SLOTS)
return;
ActorAccessibility_PlayRawSample(actor, slot, name, looping);
ActorAccessibility_ConfigureSoundForActor(actor, slot);
}
void ActorAccessibility_StopSoundForActor(AccessibleActor* actor, int slot)
@ -471,9 +539,14 @@ void ActorAccessibility_TrackNewActor(Actor* actor) {
return true;
}
void ActorAccessibility_ShutdownAudio() {
if (aa->isOn == 0)
return;
delete aa->audioEngine;
}
void ActorAccessibility_MixAccessibleAudioWithGameAudio(int16_t* ogBuffer, uint32_t nFrames) {
if (aa->isOn == 0)
return;
aa->audioEngine->mix(ogBuffer, nFrames);
}
@ -494,18 +567,48 @@ return NULL;//Resource doesn't exist, user's gotta run the extractor.
tempRecord.resource = res;
std::stringstream ss;
ss << std::setw(4) << std::setfill('0') << std::hex << sfxId;
tempRecord.hexName = ss.str();
tempRecord.path = ss.str();
aa->sfxMap[sfxId] = tempRecord;
record = &aa->sfxMap[sfxId];
aa->audioEngine->cacheDecodedSample(record->hexName, record->resource->Buffer.data(),
aa->audioEngine->cacheDecodedSample(record->path, record->resource->Buffer.data(),
record->resource->Buffer.size());
} else
record = &it->second;
return record->hexName.c_str();
return record->path.c_str();
}
//Map the path to a raw sample to the external audio engine.
const char* ActorAccessibility_MapRawSampleToExternalAudio(const char* name)
{
SfxRecord* record;
std::string key(name);
auto it = aa->sampleMap.find(key);
if (it == aa->sampleMap.end()) {
SfxRecord tempRecord;
std::stringstream ss;
ss << "audio/samples/" << key;
std::string fullPath = ss.str();
auto res = LUS::Context::GetInstance()->GetResourceManager()->LoadResource(fullPath);
if (res == nullptr)
return NULL; // Resource doesn't exist, user's gotta run the extractor.
AudioDecoder decoder;
decoder.setSample((LUS::AudioSample*)res.get());
s16* wav;
size_t wavSize = decoder.decodeToWav(&wav);
tempRecord.path = key;
tempRecord.decodedSample = std::make_shared<s16*>(wav);
aa->sampleMap[key] = tempRecord;
record = &aa->sampleMap[key];
aa->audioEngine->cacheDecodedSample(record->path, wav,
wavSize);
} else
record = &it->second;
return record->path.c_str();
}
// Call once per frame to tell the audio engine to start working on the latest batch of queued instructions.
void ActorAccessibility_PrepareNextAudioFrame() {
@ -525,5 +628,3 @@ return NULL;//Resource doesn't exist, user's gotta run the extractor.
}

View file

@ -99,6 +99,9 @@ void ActorAccessibility_PlaySpecialSound(AccessibleActor* actor, s16 sfxId);
*looping: whether to play the sound just once or on a continuous loop.
*/
void ActorAccessibility_PlaySound(void* actor, int slot, s16 sfxId, bool looping);
//Play one of the game's internal samples.
void ActorAccessibility_PlayRawSample(void* handle, int slot, const char* name, bool looping);
//
//Stop a sound. Todo: consider making this a short fade instead of just cutting it off.
void ActorAccessibility_StopSound(void* handle, int slot);
void ActorAccessibility_StopAllSounds(void* handle);
@ -121,6 +124,8 @@ void ActorAccessibility_SeekSound(void* handle, int slot, size_t offset);
*
*/
void ActorAccessibility_PlaySoundForActor(AccessibleActor* actor, int slot, s16 sfxId, bool looping);
void ActorAccessibility_PlaySampleForActor(AccessibleActor* actor, int slot, const char* name, bool looping);
void ActorAccessibility_StopSoundForActor(AccessibleActor* actor, int slot);
void ActorAccessibility_StopAllSoundsForActor(AccessibleActor* actor);
f32 ActorAccessibility_ComputeCurrentVolume(f32 maxDistance, f32 xzDistToPlayer);

View file

@ -14,9 +14,10 @@ extern f32 D_801333E0;
extern s8 D_801333E8;
extern u8 D_801333F0;
void AudioMgr_CreateNextAudioBuffer(s16* samples, u32 num_samples);
extern bool freezeGame;
}
enum {
STEP_SETUP,
STEP_SETUP = 0,
STEP_MAIN,
STEP_FINISHED,
STEP_ERROR,
@ -92,6 +93,7 @@ void SfxExtractor::renderOutput() {
}
void SfxExtractor::setup() {
try {
SpeechSynthesizer::Instance->Speak("Sfx extraction speedrun initiated. Please wait. This will take a few minutes.", GetLanguageCode());
// Kill the audio thread so we can take control.
captureThreadState = CT_WAITING;
@ -154,6 +156,8 @@ void SfxExtractor::finished() {
CVarClear("gExtractSfx");
CVarSave();
archive = nullptr;
freezeGame = false;
Audio_QueueSeqCmd(NA_BGM_TITLE);
if (currentStep == STEP_ERROR || currentStep == STEP_ERROR_OTR) {
@ -167,6 +171,7 @@ void SfxExtractor::finished() {
SpeechSynthesizer::Instance->Speak(ss.str().c_str(), GetLanguageCode());
} else
Audio_PlayFanfare(NA_BGM_ITEM_GET);
}
void SfxExtractor::maybeGiveProgressReport() {
size_t ripsRemaining = sfxToRip.size() + 1;
@ -182,7 +187,8 @@ void SfxExtractor::maybeGiveProgressReport() {
SfxExtractor::SfxExtractor() {
currentStep = STEP_SETUP;
}
void SfxExtractor::frameCallback() {
void SfxExtractor::frameCallback() {
switch (currentStep) {
case STEP_SETUP:
setup();

View file

@ -23,6 +23,7 @@ class SfxExtractor {
void maybeGiveProgressReport();
public:
SfxExtractor();
void frameCallback();
void prime();
// The below is called by the (hijacked) audio thread.

View file

@ -153,6 +153,8 @@ public:
DEFINE_HOOK(OnOcarinaSongAction, void());
DEFINE_HOOK(OnActorInit, void(void* actor));
DEFINE_HOOK(OnActorUpdate, void(void* actor));
DEFINE_HOOK(OnActorDestroy, void(void* actor));
DEFINE_HOOK(OnPlayerBonk, void());
DEFINE_HOOK(OnPlayDestroy, void());
DEFINE_HOOK(OnPlayDrawEnd, void());
@ -180,6 +182,7 @@ public:
DEFINE_HOOK(OnUpdateFileNameSelection, void(int16_t charCode));
DEFINE_HOOK(OnSetGameLanguage, void());
DEFINE_HOOK(OnGameStillFrozen, void());
// Helpers
static bool IsSaveLoaded();

View file

@ -49,8 +49,11 @@ void GameInteractor_ExecuteOnActorInit(void* actor) {
void GameInteractor_ExecuteOnActorUpdate(void* actor) {
GameInteractor::Instance->ExecuteHooks<GameInteractor::OnActorUpdate>(actor);
}
void GameInteractor_ExecuteOnActorDestroy(void* actor) {
GameInteractor::Instance->ExecuteHooks<GameInteractor::OnActorDestroy>(actor);
}
void GameInteractor_ExecuteOnPlayerBonk() {
void GameInteractor_ExecuteOnPlayerBonk() {
GameInteractor::Instance->ExecuteHooks<GameInteractor::OnPlayerBonk>();
}
@ -149,3 +152,7 @@ void GameInteractor_ExecuteOnUpdateFileNameSelection(int16_t charCode) {
void GameInteractor_ExecuteOnSetGameLanguage() {
GameInteractor::Instance->ExecuteHooks<GameInteractor::OnSetGameLanguage>();
}
void GameInteractor_ExecuteOnGameStillFrozen()
{
GameInteractor::Instance->ExecuteHooks<GameInteractor::OnGameStillFrozen>();
}

View file

@ -15,7 +15,10 @@ void GameInteractor_ExecuteOnSceneSpawnActors();
void GameInteractor_ExecuteOnPlayerUpdate();
void GameInteractor_ExecuteOnOcarinaSongAction();
void GameInteractor_ExecuteOnActorInit(void* actor);
void GameInteractor_ExecuteOnActorInit(void* actor);
void GameInteractor_ExecuteOnActorUpdate(void* actor);
void GameInteractor_ExecuteOnActorDestroy(void* actor);
void GameInteractor_ExecuteOnPlayerBonk();
void GameInteractor_ExecuteOnOcarinaSongAction();
void GameInteractor_ExecuteOnPlayDestroy();
@ -48,6 +51,7 @@ void GameInteractor_ExecuteOnUpdateFileNameSelection(int16_t charCode);
// MARK: - Game
void GameInteractor_ExecuteOnSetGameLanguage();
void GameInteractor_ExecuteOnGameStillFrozen();
#ifdef __cplusplus
}
#endif

View file

@ -113,6 +113,7 @@ CrowdControl* CrowdControl::Instance;
#include "soh/config/ConfigUpdaters.h"
#include "soh/Enhancements/accessible-actors/ActorAccessibility.h"
#include "Enhancements//accessible-actors/ActorAccessibility.h"
OTRGlobals* OTRGlobals::Instance;
SaveManager* SaveManager::Instance;
CustomMessageManager* CustomMessageManager::Instance;
@ -783,6 +784,7 @@ extern "C" void InitOTR() {
clearMtx = (uintptr_t)&gMtxClear;
OTRMessage_Init();
ActorAccessibility_Init();
OTRAudio_Init();
OTRExtScanner();
VanillaItemTable_Init();
@ -826,7 +828,7 @@ extern "C" void DeinitOTR() {
CrowdControl::Instance->Disable();
CrowdControl::Instance->Shutdown();
#endif
ActorAccessibility_Shutdown();
// Destroying gui here because we have shared ptrs to LUS objects which output to SPDLOG which is destroyed before these shared ptrs.
SohGui::Destroy();

View file

@ -3,6 +3,7 @@
#include "soh/mixer.h"
#include "soh/Enhancements/audio/AudioEditor.h"
extern bool freezeGame;
typedef struct {
u8 unk_0;
@ -371,8 +372,9 @@ extern f32 D_80130F28;
void Audio_QueueSeqCmd(u32 cmd)
{
if (CVarGetInteger("gExtractSfx", 0))
return;
if (freezeGame)
return;//No music during SFX rip.
u8 op = cmd >> 28;
if (op == 0 || op == 2 || op == 12) {
u8 seqId = cmd & 0xFF;

View file

@ -10,7 +10,7 @@
#include <libultraship/bridge.h>
#include "soh/CrashHandlerExp.h"
#include <soh/Enhancements/accessible-actors/ActorAccessibility.h>
s32 gScreenWidth = SCREEN_WIDTH;
s32 gScreenHeight = SCREEN_HEIGHT;
size_t gSystemHeapSize = 0;
@ -60,14 +60,13 @@ int main(int argc, char** argv)
#endif
GameConsole_Init();
ActorAccessibility_Init();//Needs to happen before OTR.
InitOTR();
// TODO: Was moved to below InitOTR because it requires window to be setup. But will be late to catch crashes.
CrashHandlerRegisterCallback(CrashHandler_PrintSohData);
BootCommands_Init();
Main(0);
DeinitOTR();
ActorAccessibility_Shutdown();
return 0;
}

View file

@ -77,7 +77,7 @@
#include "textures/place_title_cards/g_pn_56.h"
#include "textures/place_title_cards/g_pn_57.h"
#endif
#include <soh/Enhancements/accessible-actors/ActorAccessibility.h>
static CollisionPoly* sCurCeilingPoly;
static s32 sCurCeilingBgId;
@ -1224,6 +1224,7 @@ void Actor_Init(Actor* actor, PlayState* play) {
}
void Actor_Destroy(Actor* actor, PlayState* play) {
GameInteractor_ExecuteOnActorDestroy(actor);
if (actor->destroy != NULL) {
actor->destroy(actor, play);
actor->destroy = NULL;
@ -2476,6 +2477,7 @@ u32 D_80116068[ACTORCAT_MAX] = {
};
void Actor_UpdateAll(PlayState* play, ActorContext* actorCtx) {
Actor* refActor;
Actor* actor;
Player* player;
@ -2613,8 +2615,6 @@ void Actor_UpdateAll(PlayState* play, ActorContext* actorCtx) {
func_8002C7BC(&actorCtx->targetCtx, player, actor, play);
TitleCard_Update(play, &actorCtx->titleCtx);
DynaPoly_UpdateBgActorTransforms(play, &play->colCtx.dyna);
ActorAccessibility_RunAccessibilityForAllActors(play);
}
void Actor_FaultPrint(Actor* actor, char* command) {
@ -3215,7 +3215,6 @@ Actor* Actor_Spawn(ActorContext* actorCtx, PlayState* play, s16 actorId, f32 pos
temp = gSegments[6];
Actor_Init(actor, play);
gSegments[6] = temp;
ActorAccessibility_TrackNewActor(actor);
return actor;
}
@ -3324,9 +3323,7 @@ Actor* Actor_Delete(ActorContext* actorCtx, Actor* actor, PlayState* play) {
dbEntry->numLoaded--;
Actor_FreeOverlay(dbEntry);
ActorAccessibility_RemoveTrackedActor(actor);
return newHead;
return newHead;
}
s32 func_80032880(PlayState* play, Actor* actor) {

View file

@ -16,7 +16,7 @@
#include <time.h>
#include <assert.h>
bool freezeGame = false;//Used for SFX ripper.
void* D_8012D1F0 = NULL;
//UNK_TYPE D_8012D1F4 = 0; // unused
Input* D_8012D1F8 = NULL;
@ -753,18 +753,12 @@ void Play_Update(PlayState* play) {
Input* input;
u32 i;
s32 pad2;
//Support locking down the game on the title screen for the purposes of SFX extraction. Be careful to avoid checking CVars every frame.
static int sfxExtractionMode = -1;
if (play->sceneNum == 81) // Title screen.
{
if (sfxExtractionMode != 0)
sfxExtractionMode = CVarGetInteger("gExtractSfx", 0);
if (sfxExtractionMode == 1) {
ActorAccessibility_HandleSoundExtractionMode(play);
return;
}
if (freezeGame) {
GameInteractor_ExecuteOnGameStillFrozen();
return;
}
input = play->state.input;
if ((SREG(1) < 0) || (DREG(0) != 0)) {