Merge branch 'actor-accessibility-experiments' of https://www.github.com/caturria/shipwright into actor-accessibility-experiments

soh/soh/Enhancements/accessible-actors/AccessibleActorList.cpp

@@ -296,7 +296,10 @@ void accessible_goma(AccessibleActor* actor) {
 }
 
 void accessible_door_of_time(AccessibleActor* actor) {
-    ActorAccessibility_PlaySoundForActor(actor, 0, NA_SE_EV_DIAMOND_SWITCH, false);
+    ActorAccessibility_PlaySampleForActor(actor, 0, "Chanting", false);
+    ActorAccessibility_SetSoundPitch(actor, 0, 1.0);
+
+    //ActorAccessibility_PlaySoundForActor(actor, 0, NA_SE_EV_DIAMOND_SWITCH, false);
 }
 
 void accessible_sticks(AccessibleActor* actor) {

soh/soh/Enhancements/accessible-actors/ActorAccessibility.cpp

@@ -16,6 +16,12 @@
 #include <unordered_set>
 #include "soh/Enhancements/speechsynthesizer/SpeechSynthesizer.h"
 #include "soh/Enhancements/tts/tts.h"
+#include "soh/Enhancements/game-interactor/GameInteractor.h"
+#include "soh/Enhancements/audio/AudioDecoder.h"
+extern "C" {
+extern PlayState* gPlayState;
+extern bool freezeGame;
+}
 
 const char* GetLanguageCode();
 
@@ -43,14 +49,16 @@ typedef std::map<s32, VAList_t> VAZones_t;//Maps room/ scene indices to their co
 typedef std::unordered_set<s16> SceneList_t;//A list of scenes which have already been visited (since the game was launched). Used to prevent re-creation of terrain VAs every time the player reloads a scene.
 
 typedef struct {
-    std::string hexName;
+    std::string path;
     std::shared_ptr<LUS::File> resource;
+    std::shared_ptr<s16*> decodedSample;//Set if the record is for a raw sample as opposed to a SFX.
+
 }SfxRecord;
 
 class ActorAccessibility {
   public:
-      bool isOn = false;
-    uint64_t nextActorID;
+      int isOn = 0;
+    uint64_t nextActorID = 0;
     SupportedActors_t supportedActors;
     TrackedActors_t trackedActors;
     AccessibleActorList_t accessibleActorList;
@@ -59,7 +67,8 @@ class ActorAccessibility {
     AccessibleAudioEngine* audioEngine;
     SfxExtractor sfxExtractor;
     std::unordered_map<s16, SfxRecord> sfxMap;//Maps internal sfx to external (prerendered) resources.
-
+    std::unordered_map<std::string, SfxRecord> sampleMap;//Similar to above, but this one maps raw audio samples as opposed to SFX.
+    int extractSfx = 0;
 };
 static ActorAccessibility* aa;
 
@@ -68,12 +77,48 @@ uint64_t ActorAccessibility_GetNextID() {
     aa->nextActorID++;
     return result;
 }
-void ActorAccessibility_Init() {
-    aa = new ActorAccessibility();
-    ActorAccessibility_InitAudio();
-    ActorAccessibility_InitActors();
+
+// Hooks for game-interactor.
+void ActorAccessibility_OnActorInit(void* actor) {
+    ActorAccessibility_TrackNewActor((Actor*)actor);
+}
+void ActorAccessibility_OnGameFrameUpdate() {
+    if (gPlayState == NULL)
+        return;
+
+    ActorAccessibility_RunAccessibilityForAllActors(gPlayState);
+}
+void ActorAccessibility_OnActorDestroy(void* actor)
+{
+    ActorAccessibility_RemoveTrackedActor((Actor*) actor);
 
 }
+void ActorAccessibility_OnGameStillFrozen()
+{
+    if (gPlayState == NULL)
+        return;
+    if (aa->extractSfx)
+        ActorAccessibility_HandleSoundExtractionMode(gPlayState);
+
+}
+    void ActorAccessibility_Init() {
+
+    aa = new ActorAccessibility();
+    aa->isOn = CVarGetInteger("gA11yAudioInteraction", 0);
+    if (!aa->isOn)
+        return;
+    aa->extractSfx = CVarGetInteger("gExtractSfx", 0);
+    if (aa->extractSfx)
+        freezeGame = true;
+    ActorAccessibility_InitAudio();
+    ActorAccessibility_InitActors();
+    GameInteractor::Instance->RegisterGameHook<GameInteractor::OnActorInit>(ActorAccessibility_OnActorInit);
+    GameInteractor::Instance->RegisterGameHook<GameInteractor::OnActorDestroy>(ActorAccessibility_OnActorDestroy);
+
+    GameInteractor::Instance->RegisterGameHook<GameInteractor::OnGameFrameUpdate>(ActorAccessibility_OnGameFrameUpdate);
+    GameInteractor::Instance->RegisterGameHook<GameInteractor::OnGameStillFrozen>(ActorAccessibility_OnGameStillFrozen);
+
+    }
 void ActorAccessibility_Shutdown() {
     ActorAccessibility_ShutdownAudio();
     delete aa;
@@ -88,7 +133,7 @@ void ActorAccessibility_Shutdown() {
     policy->pitch = 1.5;
     policy->runsAlways = false;
     policy->sound = sfx;
-    policy->volume = 1.0;
+    policy->volume = 0.5;
     policy->initUserData = NULL;
     policy->cleanupUserData = NULL;
     policy->pitchModifier = 0.1;
@@ -114,7 +159,7 @@ int ActorAccessibility_GetRandomStartingFrameCount(int min, int max) {
 
  }
 
-void ActorAccessibility_TrackNewActor(Actor* actor) {
+    void ActorAccessibility_TrackNewActor(Actor * actor) {
         // Don't track actors for which no accessibility policy has been configured.
         ActorAccessibilityPolicy* policy = ActorAccessibility_GetPolicyForActor(actor->id);
         if (policy == NULL)
@@ -178,12 +223,23 @@ void ActorAccessibility_TrackNewActor(Actor* actor) {
         Audio_PlaySoundGeneral(sfxId, &actor->projectedPos, 4, &actor->currentPitch, &actor->currentVolume, &actor->currentReverb);
     }
     const char* ActorAccessibility_MapSfxToExternalAudio(s16 sfxId);
-
     void ActorAccessibility_PlaySound(void* handle, int slot, s16 sfxId, bool looping)
     {
         const char* path = ActorAccessibility_MapSfxToExternalAudio(sfxId);
+        if (path == NULL)
+            return;
         aa->audioEngine->playSound((uintptr_t)handle, slot, path, looping);
     }
+    const char* ActorAccessibility_MapRawSampleToExternalAudio(const char* name);
+
+    void ActorAccessibility_PlayRawSample(void* handle, int slot, const char* name, bool looping)
+    {
+        const char* path = ActorAccessibility_MapRawSampleToExternalAudio(name);
+        if (path == NULL)
+            return;
+        aa->audioEngine->playSound((uintptr_t)handle, slot, path, looping);
+
+    }
     void ActorAccessibility_StopSound(void* handle, int slot)
     {
         aa->audioEngine->stopSound((uintptr_t) handle, slot);
@@ -231,17 +287,29 @@ void ActorAccessibility_TrackNewActor(Actor* actor) {
         aa->audioEngine->seekSound((uintptr_t)handle, slot, offset);
 
     }
+    void ActorAccessibility_ConfigureSoundForActor(AccessibleActor* actor, int slot)
+    {
+        ActorAccessibility_SetSoundPitch(actor, slot, actor->policy.pitch);
+        ActorAccessibility_SetPitchBehindModifier(actor, slot, actor->policy.pitchModifier);
+        ActorAccessibility_SetSoundPos(actor, slot, &actor->projectedPos, actor->xyzDistToPlayer,
+                                       actor->policy.distance);
+        ActorAccessibility_SetSoundVolume(actor, slot, actor->policy.volume);
+        actor->managedSoundSlots[slot] = true;
+    }
     void ActorAccessibility_PlaySoundForActor(AccessibleActor* actor, int slot, s16 sfxId, bool looping)
     {
         if (slot < 0 || slot > NUM_MANAGED_SOUND_SLOTS)
             return;
         ActorAccessibility_PlaySound(actor, slot, sfxId, looping);
-        ActorAccessibility_SetSoundPitch(actor, slot, actor->policy.pitch);
-        ActorAccessibility_SetPitchBehindModifier(actor, slot, actor->policy.pitchModifier);
-        ActorAccessibility_SetSoundPos(actor, slot, &actor->projectedPos, actor->xzDistToPlayer,
-                                       actor->policy.distance);
-        ActorAccessibility_SetSoundVolume(actor, slot, actor->policy.volume);
-        actor->managedSoundSlots[slot] = true;
+        ActorAccessibility_ConfigureSoundForActor(actor, slot);
+
+    }
+    void ActorAccessibility_PlaySampleForActor(AccessibleActor* actor, int slot, const char* name, bool looping)
+    {
+        if (slot < 0 || slot > NUM_MANAGED_SOUND_SLOTS)
+            return;
+        ActorAccessibility_PlayRawSample(actor, slot, name, looping);
+        ActorAccessibility_ConfigureSoundForActor(actor, slot);
 
     }
     void ActorAccessibility_StopSoundForActor(AccessibleActor* actor, int slot)
@@ -471,9 +539,14 @@ void ActorAccessibility_TrackNewActor(Actor* actor) {
         return true;
     }
     void ActorAccessibility_ShutdownAudio() {
+        if (aa->isOn == 0)
+            return;
         delete aa->audioEngine;
     }
     void ActorAccessibility_MixAccessibleAudioWithGameAudio(int16_t* ogBuffer, uint32_t nFrames) {
+        if (aa->isOn == 0)
+            return;
+
         aa->audioEngine->mix(ogBuffer, nFrames);
 
     }
@@ -494,18 +567,48 @@ return NULL;//Resource doesn't exist, user's gotta run the extractor.
             tempRecord.resource = res;
             std::stringstream ss;
             ss << std::setw(4) << std::setfill('0') << std::hex << sfxId;
-            tempRecord.hexName = ss.str();
+            tempRecord.path = ss.str();
             aa->sfxMap[sfxId] = tempRecord;
             record = &aa->sfxMap[sfxId];
-            aa->audioEngine->cacheDecodedSample(record->hexName, record->resource->Buffer.data(),
+            aa->audioEngine->cacheDecodedSample(record->path, record->resource->Buffer.data(),
                                                 record->resource->Buffer.size());
         } else
             record = &it->second;
 
-        return record->hexName.c_str();
+        return record->path.c_str();
 
 
     }
+//Map the path to a raw sample to the external audio engine.
+    const char* ActorAccessibility_MapRawSampleToExternalAudio(const char* name)
+    {
+        SfxRecord* record;
+        std::string key(name);
+        auto it = aa->sampleMap.find(key);
+        if (it == aa->sampleMap.end()) {
+            SfxRecord tempRecord;
+            std::stringstream ss;
+            ss << "audio/samples/" << key;
+            std::string fullPath = ss.str();
+            auto res = LUS::Context::GetInstance()->GetResourceManager()->LoadResource(fullPath);
+            if (res == nullptr)
+return NULL; // Resource doesn't exist, user's gotta run the extractor.
+            AudioDecoder decoder;
+            decoder.setSample((LUS::AudioSample*)res.get());
+            s16* wav;
+            size_t wavSize = decoder.decodeToWav(&wav);
+
+            tempRecord.path = key;
+            tempRecord.decodedSample = std::make_shared<s16*>(wav);
+            aa->sampleMap[key] = tempRecord;
+            record = &aa->sampleMap[key];
+            aa->audioEngine->cacheDecodedSample(record->path, wav,
+                                                wavSize);
+        } else
+            record = &it->second;
+
+        return record->path.c_str();
+    }
     // Call once per frame to tell the audio engine to start working on the latest batch of queued instructions.
 
     void ActorAccessibility_PrepareNextAudioFrame() {
@@ -525,5 +628,3 @@ return NULL;//Resource doesn't exist, user's gotta run the extractor.
     }
 
 
-  
-    

soh/soh/Enhancements/accessible-actors/ActorAccessibility.h

@@ -99,6 +99,9 @@ void ActorAccessibility_PlaySpecialSound(AccessibleActor* actor, s16 sfxId);
  *looping: whether to play the sound just once or on a continuous loop.
 */
 void ActorAccessibility_PlaySound(void* actor, int slot, s16 sfxId, bool looping);
+//Play one of the game's internal samples.
+ void ActorAccessibility_PlayRawSample(void* handle, int slot, const char* name, bool looping);
+// 
 //Stop a sound. Todo: consider making this a short fade instead of just cutting it off.
 void ActorAccessibility_StopSound(void* handle, int slot);
 void ActorAccessibility_StopAllSounds(void* handle);
@@ -121,6 +124,8 @@ void ActorAccessibility_SeekSound(void* handle, int slot, size_t offset);
 * 
 */
 void ActorAccessibility_PlaySoundForActor(AccessibleActor* actor, int slot, s16 sfxId, bool looping);
+void ActorAccessibility_PlaySampleForActor(AccessibleActor* actor, int slot, const char* name, bool looping);
+
 void ActorAccessibility_StopSoundForActor(AccessibleActor* actor, int slot);
 void ActorAccessibility_StopAllSoundsForActor(AccessibleActor* actor);
 f32 ActorAccessibility_ComputeCurrentVolume(f32 maxDistance, f32 xzDistToPlayer);

soh/soh/Enhancements/accessible-actors/SfxExtractor.cpp

@@ -14,9 +14,10 @@ extern f32 D_801333E0;
 extern s8 D_801333E8;
 extern u8 D_801333F0;
 void AudioMgr_CreateNextAudioBuffer(s16* samples, u32 num_samples);
+extern bool freezeGame;
 }
 enum {
-    STEP_SETUP,
+    STEP_SETUP = 0,
     STEP_MAIN,
     STEP_FINISHED,
     STEP_ERROR,
@@ -92,6 +93,7 @@ void SfxExtractor::renderOutput() {
 }
 void SfxExtractor::setup() {
     try {
+
         SpeechSynthesizer::Instance->Speak("Sfx extraction speedrun initiated. Please wait. This will take a few minutes.", GetLanguageCode());
         // Kill the audio thread so we can take control.
         captureThreadState = CT_WAITING;
@@ -154,6 +156,8 @@ void SfxExtractor::finished() {
     CVarClear("gExtractSfx");
     CVarSave();
     archive = nullptr;
+    freezeGame = false;
+
     Audio_QueueSeqCmd(NA_BGM_TITLE);
 
     if (currentStep == STEP_ERROR || currentStep == STEP_ERROR_OTR) {
@@ -167,6 +171,7 @@ void SfxExtractor::finished() {
         SpeechSynthesizer::Instance->Speak(ss.str().c_str(), GetLanguageCode());
     } else
         Audio_PlayFanfare(NA_BGM_ITEM_GET);
+
 }
 void SfxExtractor::maybeGiveProgressReport() {
     size_t ripsRemaining = sfxToRip.size() + 1;
@@ -182,7 +187,8 @@ void SfxExtractor::maybeGiveProgressReport() {
 SfxExtractor::SfxExtractor() {
     currentStep = STEP_SETUP;
 }
-void SfxExtractor::frameCallback() {
+
+    void SfxExtractor::frameCallback() {
     switch (currentStep) {
         case STEP_SETUP:
             setup();

soh/soh/Enhancements/accessible-actors/SfxExtractor.h

@@ -23,6 +23,7 @@ class SfxExtractor {
     void maybeGiveProgressReport();
   public:
     SfxExtractor();
+
     void frameCallback();
     void prime();
     // The below is called by the (hijacked) audio thread.

soh/soh/Enhancements/game-interactor/GameInteractor.h

@@ -153,6 +153,8 @@ public:
     DEFINE_HOOK(OnOcarinaSongAction, void());
     DEFINE_HOOK(OnActorInit, void(void* actor));
     DEFINE_HOOK(OnActorUpdate, void(void* actor));
+    DEFINE_HOOK(OnActorDestroy, void(void* actor));
+
     DEFINE_HOOK(OnPlayerBonk, void());
     DEFINE_HOOK(OnPlayDestroy, void());
     DEFINE_HOOK(OnPlayDrawEnd, void());
@@ -180,6 +182,7 @@ public:
     DEFINE_HOOK(OnUpdateFileNameSelection, void(int16_t charCode));
     
     DEFINE_HOOK(OnSetGameLanguage, void());
+    DEFINE_HOOK(OnGameStillFrozen, void());
 
     // Helpers
     static bool IsSaveLoaded();

soh/soh/Enhancements/game-interactor/GameInteractor_Hooks.cpp

@@ -49,8 +49,11 @@ void GameInteractor_ExecuteOnActorInit(void* actor) {
 void GameInteractor_ExecuteOnActorUpdate(void* actor) {
     GameInteractor::Instance->ExecuteHooks<GameInteractor::OnActorUpdate>(actor);
 }
+void GameInteractor_ExecuteOnActorDestroy(void* actor) {
+    GameInteractor::Instance->ExecuteHooks<GameInteractor::OnActorDestroy>(actor);
+}
 
-void GameInteractor_ExecuteOnPlayerBonk() {
+    void GameInteractor_ExecuteOnPlayerBonk() {
     GameInteractor::Instance->ExecuteHooks<GameInteractor::OnPlayerBonk>();
 }
 
@@ -149,3 +152,7 @@ void GameInteractor_ExecuteOnUpdateFileNameSelection(int16_t charCode) {
 void GameInteractor_ExecuteOnSetGameLanguage() {
     GameInteractor::Instance->ExecuteHooks<GameInteractor::OnSetGameLanguage>();
 }
+void GameInteractor_ExecuteOnGameStillFrozen()
+{
+    GameInteractor::Instance->ExecuteHooks<GameInteractor::OnGameStillFrozen>();
+}

soh/soh/Enhancements/game-interactor/GameInteractor_Hooks.h

@@ -15,7 +15,10 @@ void GameInteractor_ExecuteOnSceneSpawnActors();
 void GameInteractor_ExecuteOnPlayerUpdate();
 void GameInteractor_ExecuteOnOcarinaSongAction();
 void GameInteractor_ExecuteOnActorInit(void* actor);
+void GameInteractor_ExecuteOnActorInit(void* actor);
 void GameInteractor_ExecuteOnActorUpdate(void* actor);
+void GameInteractor_ExecuteOnActorDestroy(void* actor);
+
 void GameInteractor_ExecuteOnPlayerBonk();
 void GameInteractor_ExecuteOnOcarinaSongAction();
 void GameInteractor_ExecuteOnPlayDestroy();
@@ -48,6 +51,7 @@ void GameInteractor_ExecuteOnUpdateFileNameSelection(int16_t charCode);
 
 // MARK: - Game
 void GameInteractor_ExecuteOnSetGameLanguage();
+void GameInteractor_ExecuteOnGameStillFrozen();
 #ifdef __cplusplus
 }
 #endif

soh/soh/OTRGlobals.cpp

@@ -113,6 +113,7 @@ CrowdControl* CrowdControl::Instance;
 
 #include "soh/config/ConfigUpdaters.h"
 #include "soh/Enhancements/accessible-actors/ActorAccessibility.h"
+#include "Enhancements//accessible-actors/ActorAccessibility.h"
 OTRGlobals* OTRGlobals::Instance;
 SaveManager* SaveManager::Instance;
 CustomMessageManager* CustomMessageManager::Instance;
@@ -783,6 +784,7 @@ extern "C" void InitOTR() {
     
     clearMtx = (uintptr_t)&gMtxClear;
     OTRMessage_Init();
+    ActorAccessibility_Init();
     OTRAudio_Init();
     OTRExtScanner();
     VanillaItemTable_Init();
@@ -826,7 +828,7 @@ extern "C" void DeinitOTR() {
     CrowdControl::Instance->Disable();
     CrowdControl::Instance->Shutdown();
 #endif
-
+    ActorAccessibility_Shutdown();
     // Destroying gui here because we have shared ptrs to LUS objects which output to SPDLOG which is destroyed before these shared ptrs.
     SohGui::Destroy();
 

soh/src/code/code_800F9280.c

@@ -3,6 +3,7 @@
 #include "soh/mixer.h"
 
 #include "soh/Enhancements/audio/AudioEditor.h"
+extern bool freezeGame;
 
 typedef struct {
     u8 unk_0;
@@ -371,8 +372,9 @@ extern f32 D_80130F28;
 
 void Audio_QueueSeqCmd(u32 cmd) 
 {
-    if (CVarGetInteger("gExtractSfx", 0))
-        return;
+    if (freezeGame)
+        return;//No music during SFX rip.
+
     u8 op = cmd >> 28;
     if (op == 0 || op == 2 || op == 12) {
         u8 seqId = cmd & 0xFF;

soh/src/code/main.c

@@ -10,7 +10,7 @@
 
 #include <libultraship/bridge.h>
 #include "soh/CrashHandlerExp.h"
-#include <soh/Enhancements/accessible-actors/ActorAccessibility.h>
+
 s32 gScreenWidth = SCREEN_WIDTH;
 s32 gScreenHeight = SCREEN_HEIGHT;
 size_t gSystemHeapSize = 0;
@@ -60,14 +60,13 @@ int main(int argc, char** argv)
 #endif
 
     GameConsole_Init();
-    ActorAccessibility_Init();//Needs to happen before OTR.
     InitOTR();
     // TODO: Was moved to below InitOTR because it requires window to be setup. But will be late to catch crashes.
     CrashHandlerRegisterCallback(CrashHandler_PrintSohData);
     BootCommands_Init();
+
     Main(0);
     DeinitOTR();
-    ActorAccessibility_Shutdown();
     return 0;
 }
 

soh/src/code/z_actor.c

@@ -77,7 +77,7 @@
 #include "textures/place_title_cards/g_pn_56.h"
 #include "textures/place_title_cards/g_pn_57.h"
 #endif
-#include <soh/Enhancements/accessible-actors/ActorAccessibility.h>
+
 static CollisionPoly* sCurCeilingPoly;
 static s32 sCurCeilingBgId;
 
@@ -1224,6 +1224,7 @@ void Actor_Init(Actor* actor, PlayState* play) {
 }
 
 void Actor_Destroy(Actor* actor, PlayState* play) {
+    GameInteractor_ExecuteOnActorDestroy(actor);
     if (actor->destroy != NULL) {
         actor->destroy(actor, play);
         actor->destroy = NULL;
@@ -2476,6 +2477,7 @@ u32 D_80116068[ACTORCAT_MAX] = {
 };
 
 void Actor_UpdateAll(PlayState* play, ActorContext* actorCtx) {
+
     Actor* refActor;
     Actor* actor;
     Player* player;
@@ -2613,8 +2615,6 @@ void Actor_UpdateAll(PlayState* play, ActorContext* actorCtx) {
     func_8002C7BC(&actorCtx->targetCtx, player, actor, play);
     TitleCard_Update(play, &actorCtx->titleCtx);
     DynaPoly_UpdateBgActorTransforms(play, &play->colCtx.dyna);
-    ActorAccessibility_RunAccessibilityForAllActors(play);
-
 }
 
 void Actor_FaultPrint(Actor* actor, char* command) {
@@ -3215,7 +3215,6 @@ Actor* Actor_Spawn(ActorContext* actorCtx, PlayState* play, s16 actorId, f32 pos
     temp = gSegments[6];
     Actor_Init(actor, play);
     gSegments[6] = temp;
-    ActorAccessibility_TrackNewActor(actor);
 
     return actor;
 }
@@ -3324,8 +3323,6 @@ Actor* Actor_Delete(ActorContext* actorCtx, Actor* actor, PlayState* play) {
     dbEntry->numLoaded--;
     Actor_FreeOverlay(dbEntry);
 
-ActorAccessibility_RemoveTrackedActor(actor);
-
     return newHead;
 }
 

soh/src/code/z_play.c

@@ -16,7 +16,7 @@
 
 #include <time.h>
 #include <assert.h>
-
+bool freezeGame = false;//Used for SFX ripper.
 void* D_8012D1F0 = NULL;
 //UNK_TYPE D_8012D1F4 = 0; // unused
 Input* D_8012D1F8 = NULL;
@@ -753,17 +753,11 @@ void Play_Update(PlayState* play) {
     Input* input;
     u32 i;
     s32 pad2;
-    //Support locking down the game on the title screen for the purposes of SFX extraction. Be careful to avoid checking CVars every frame.
-    static int sfxExtractionMode = -1;
-    if (play->sceneNum == 81) // Title screen.
-    {
-        if (sfxExtractionMode != 0)
-            sfxExtractionMode = CVarGetInteger("gExtractSfx", 0);
-        if (sfxExtractionMode == 1) {
-            ActorAccessibility_HandleSoundExtractionMode(play);
+    if (freezeGame) {
+        GameInteractor_ExecuteOnGameStillFrozen();
         return;
     }
-    }
+
 
     input = play->state.input;