Merge d222dbc127 into 237dcfec74

soh/CMakeLists.txt

@@ -161,6 +161,11 @@ if (NOT ESPEAK)
     list(FILTER soh__ EXCLUDE REGEX "soh/Enhancements/speechsynthesizer/ESpeak")
 endif()
 
+# handle accessible audio engine removals
+if (CMAKE_SYSTEM_NAME MATCHES "NintendoSwitch|CafeOS")
+    list(FILTER soh__Enhancements EXCLUDE REGEX "soh/Enhancements/accessible-actors/")
+endif()
+
 # soh/Extractor {{{
 list(FILTER ship__ EXCLUDE REGEX "soh/Extractor/*")
 
@@ -294,6 +299,14 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(dr_libs)
 
+FetchContent_Declare(
+        miniaudio
+        GIT_REPOSITORY https://github.com/mackron/miniaudio.git
+        GIT_TAG 350784a9467a79d0fa65802132668e5afbcf3777 
+	SOURCE_SUBDIR "ignore CMakeLists.txt"
+)
+FetchContent_MakeAvailable(miniaudio)
+
 find_package(SDL2)
 set(SDL2-INCLUDE ${SDL2_INCLUDE_DIRS})
 
@@ -349,6 +362,7 @@ target_include_directories(${PROJECT_NAME} PRIVATE assets
     ${SDL2-NET-INCLUDE}
 	${CMAKE_CURRENT_SOURCE_DIR}/assets/
 	${dr_libs_SOURCE_DIR}
+	${miniaudio_SOURCE_DIR}
 	.
 )
 

soh/soh/Enhancements/accessible-actors/AccessibleAudioEngine.cpp

@@ -0,0 +1,614 @@
+#define AAE_CHANNELS 2
+#define AAE_SAMPLE_RATE 44100
+#define AAE_MAX_BUFFER_SIZE AAE_SAMPLE_RATE / 10
+#define AAE_PREP_CHUNK_SIZE 64
+#define AAE_MIX_CHUNK_SIZE 64
+#define AAE_GC_INTERVAL 20 * 60 // How often, in frames, do we clean up sound handles that are no longer active.
+#define AAE_MAX_DB_REDUCTION -20
+#define AAE_LPF_ORDER 4
+
+#define NOMINMAX // because Windows is a joke.
+#define MINIAUDIO_IMPLEMENTATION
+#include "AccessibleAudioEngine.h"
+
+extern "C" {
+int AudioPlayer_GetDesiredBuffered();
+}
+#include <math.h>
+#include <algorithm>
+#include <stdexcept>
+
+enum AAE_COMMANDS {
+    AAE_START = 0,
+    AAE_STOP,
+    AAE_STOP_ALL,
+    AAE_PITCH,
+    AAE_PITCH_BEHIND, // Specify how much to change the pitch when the sound is behind the listener.
+    AAE_VOLUME,
+    AAE_PAN,
+    AAE_FILTER,
+    AAE_SEEK,
+    AAE_POS,
+    AAE_PREPARE,
+    AAE_TERMINATE,
+};
+typedef int8_t s8;
+typedef uint8_t u8;
+// Processing for our custom audio positioning.
+static float lerp_aae(float x, float y, float z) {
+    return (1.0 - z) * x + z * y;
+}
+
+static float computeGain(SoundExtras* extras) {
+    if (extras->maxDistance == 0)
+        return 0;
+    float leftover = ma_volume_db_to_linear(AAE_MAX_DB_REDUCTION);
+    float normDist = fabs(extras->distToPlayer) / extras->maxDistance;
+    float db = lerp_aae(0, AAE_MAX_DB_REDUCTION, normDist);
+    float gain = ma_volume_db_to_linear(db);
+    gain -= lerp_aae(0, leftover, normDist);
+    return gain;
+}
+// Borrow pan calculation from game itself. Todo: this is technical debt, so copy/revise or something
+extern "C" int8_t Audio_ComputeSoundPanSigned(float x, float z, uint8_t token);
+static void positioner_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn,
+                                          float** ppFramesOut, ma_uint32* pFrameCountOut) {
+
+    const float* framesIn = ppFramesIn[0];
+    float* framesOut = ppFramesOut[0];
+    ma_copy_pcm_frames(framesOut, framesIn, *pFrameCountIn, ma_format_f32, 2);
+    *pFrameCountOut = *pFrameCountIn;
+    SoundExtras* extras = (SoundExtras*)pNode;
+    // Pan the sound based on its projected position.
+    float pan;
+    // Use the game's panning mechanism, which returns a signed 8-bit integer between 0 (far-left) and 127 (far-right).
+    // It would appear that the correct thing to do is interpret this value as a gain factor in decibels. In practice,
+    // values below 38 or above 90 are never seen, so a sound that's panned far to one side or the other amounts to
+    // about -25DB worth of attenuation. Also: lie about the value of Z and give it a constant value to prevent weird
+    // behaviour when Z is far away.
+    s8 panSigned = Audio_ComputeSoundPanSigned(extras->x, extras->z, 4);
+    int db;
+    if (panSigned < 64)
+        db = 64 - panSigned;
+    else
+        db = panSigned - 64;
+    pan = 1.0 - fabs(ma_volume_db_to_linear(-db / 2));
+    if (panSigned < 64)
+        pan = -pan;
+
+    ma_panner_set_pan(&extras->panner, pan);
+    ma_panner_process_pcm_frames(&extras->panner, framesOut, framesOut, *pFrameCountIn);
+    // Next we'll apply the gain based on the object's distance relationship to the player. The strategy here is to use
+    // a combination of decibel-based and linear attenuation, so that the gain reaches 0 at the exact point when the
+    // object is at exactly the maximum distance from the player.
+
+    float gain = computeGain(extras);
+    ma_gainer_set_gain(&extras->gainer, gain);
+    ma_gainer_process_pcm_frames(&extras->gainer, framesOut, framesOut, *pFrameCountIn);
+    // Run LPF only when necessary because we can't afford to run a 4th-order lowpass on every single sound. This
+    // probably causes minor glitches when the filter switches on and off. Todo: cross that bridge.
+    if (extras->cutoff != 1.0)
+        ma_lpf_process_pcm_frames(&extras->filter, framesOut, framesOut, *pFrameCountIn);
+}
+
+static ma_node_vtable positioner_vtable = { positioner_process_pcm_frames, NULL, 1, 1, 0 };
+static ma_uint32 positioner_channels[1] = { 2 };
+
+void AccessibleAudioEngine::destroy() {
+    switch (initialized) {
+        case 3:
+            ma_engine_uninit(&engine);
+        case 2:
+            ma_pcm_rb_uninit(&preparedOutput);
+        case 1:
+            ma_resource_manager_uninit(&resourceManager);
+    }
+}
+
+void AccessibleAudioEngine::destroyAndThrow(const char* exceptionText) {
+    destroy();
+    throw std::runtime_error(exceptionText);
+}
+
+uint32_t AccessibleAudioEngine::retrieve(float* buffer, uint32_t nFrames) {
+    uint32_t framesAvailable = ma_pcm_rb_available_read(&preparedOutput);
+    if (nFrames > framesAvailable)
+        nFrames = framesAvailable;
+    if (nFrames == 0)
+        return 0;
+    uint32_t ogNFrames = nFrames;
+    while (nFrames > 0) {
+        void* readBuffer;
+        uint32_t framesObtained = nFrames;
+        ma_pcm_rb_acquire_read(&preparedOutput, &framesObtained, (void**)&readBuffer);
+        if (framesObtained > nFrames)
+            framesObtained = nFrames;
+
+        memcpy(buffer, readBuffer, sizeof(float) * framesObtained * AAE_CHANNELS);
+        buffer += framesObtained * AAE_CHANNELS;
+        nFrames -= framesObtained;
+        ma_pcm_rb_commit_read(&preparedOutput, framesObtained);
+    }
+
+    return ogNFrames;
+}
+
+void AccessibleAudioEngine::doPrepare(SoundAction& action) {
+    framesUntilGC--;
+    int nFrames = ma_pcm_rb_available_write(&preparedOutput);
+    if (nFrames <= 0)
+        return;
+
+    float* chunk;
+    while (nFrames > 0) {
+        // This should not loop more than twice.
+        uint32_t nextChunk = nFrames;
+        ma_pcm_rb_acquire_write(&preparedOutput, &nextChunk,
+                                (void**)&chunk); // Might reduce nextChunk if there isn't enough buffer space available
+                                                 // to accommodate the request.
+        ma_uint64 framesRead = 0;
+        ma_engine_read_pcm_frames(&engine, chunk, nextChunk, &framesRead);
+        // Even if we get fewer frames than expected, we should still submit a full buffer of silence.
+        if (framesRead < nextChunk)
+            ma_silence_pcm_frames(chunk + (framesRead * 2), (nextChunk - framesRead), ma_format_f32, 2);
+        ma_pcm_rb_commit_write(&preparedOutput, nextChunk);
+        nFrames -= nextChunk;
+    }
+}
+int AccessibleAudioEngine::getSoundActions(SoundAction* dest, int limit) {
+    std::unique_lock<std::mutex> lock(mtx);
+    while (soundActions.empty())
+        cv.wait(lock);
+    int actionsOut = 0;
+    while (!soundActions.empty() && limit > 0) {
+        dest[actionsOut] = soundActions.front();
+        soundActions.pop_front();
+        actionsOut++;
+        limit--;
+    }
+    return actionsOut;
+}
+void AccessibleAudioEngine::postSoundActions() {
+    {
+        std::scoped_lock<std::mutex> lock(mtx);
+        for (int i = 0; i < nextOutgoingSoundAction; i++)
+            soundActions.push_back(outgoingSoundActions[i]);
+    }
+    cv.notify_one();
+    nextOutgoingSoundAction = 0;
+}
+void AccessibleAudioEngine::postHighPrioritySoundAction(SoundAction& action) {
+    std::scoped_lock<std::mutex> lock(mtx);
+    soundActions.push_front(action);
+    cv.notify_one();
+}
+SoundAction& AccessibleAudioEngine::getNextOutgoingSoundAction() {
+    if (nextOutgoingSoundAction >= AAE_SOUND_ACTION_BATCH_SIZE)
+        postSoundActions();
+    nextOutgoingSoundAction++;
+    return outgoingSoundActions[nextOutgoingSoundAction - 1];
+}
+void AccessibleAudioEngine::runThread() {
+    bool shouldTerminate = false;
+    SoundAction incomingSoundActions[AAE_SOUND_ACTION_BATCH_SIZE];
+    while (true) {
+        processAudioJobs();
+        if (framesUntilGC <= 0)
+            garbageCollect();
+
+        int batchSize = getSoundActions(incomingSoundActions, AAE_SOUND_ACTION_BATCH_SIZE);
+        for (int i = 0; i < batchSize; i++) {
+            SoundAction& action = incomingSoundActions[i];
+            switch (action.command) {
+                case AAE_TERMINATE:
+                    return;
+                case AAE_START:
+                    doPlaySound(action);
+                    break;
+                case AAE_STOP:
+                    doStopSound(action);
+                    break;
+                case AAE_STOP_ALL:
+                    doStopAllSounds(action);
+                    break;
+                case AAE_PITCH:
+                    doSetPitch(action);
+                    break;
+                case AAE_PITCH_BEHIND:
+                    doSetPitchBehindModifier(action);
+                    break;
+                case AAE_VOLUME:
+                    doSetVolume(action);
+                    break;
+                case AAE_PAN:
+                    doSetPan(action);
+                    break;
+                case AAE_FILTER:
+                    doSetFilter(action);
+                    break;
+                case AAE_SEEK:
+                    doSeekSound(action);
+                    break;
+                case AAE_POS:
+                    doSetSoundPos(action);
+                    break;
+                case AAE_PREPARE:
+                    doPrepare(action);
+                    break;
+            }
+        }
+    }
+}
+
+SoundSlot* AccessibleAudioEngine::findSound(SoundAction& action) {
+    if (action.slot >= AAE_SLOTS_PER_HANDLE)
+        return NULL;
+    auto i = sounds.find(action.handle);
+    if (i == sounds.end())
+        return NULL;
+    SoundSlot& target = i->second[action.slot];
+    if (!target.active)
+        return NULL;
+    return &target;
+}
+
+void AccessibleAudioEngine::doPlaySound(SoundAction& action) {
+    SoundSlot* sound;
+    if (sounds.contains(action.handle)) {
+        sound = &sounds[action.handle][action.slot];
+        if (sound->active) {
+            ma_sound_stop(&sound->sound);
+            destroySound(sound);
+        }
+    } else {
+        SoundSlots temp;
+        for (int i = 0; i < AAE_SLOTS_PER_HANDLE; i++)
+            temp[i].active = false;
+
+        sounds[action.handle] = temp;
+        sound = &sounds[action.handle][action.slot];
+    }
+
+    ma_result result = ma_sound_init_from_file(&engine, action.path.c_str(),
+                                               MA_SOUND_FLAG_NO_SPATIALIZATION | MA_SOUND_FLAG_NO_DEFAULT_ATTACHMENT,
+                                               NULL, NULL, &sound->sound);
+    if (result != MA_SUCCESS) {
+        return;
+    }
+
+    if (action.handle != 0) {
+        initSoundExtras(sound);
+        // We actually attach the extras to the engine, not the sound itself.
+        ma_node_attach_output_bus(&sound->extras, 0, ma_node_graph_get_endpoint(&engine.nodeGraph), 0);
+    } else {
+        sound->extras.base.pNodeGraph = NULL;
+        ma_node_attach_output_bus(&sound->sound, 0, ma_node_graph_get_endpoint(&engine.nodeGraph), 0);
+    }
+
+    ma_sound_start(&sound->sound);
+
+    sound->active = true;
+}
+
+void AccessibleAudioEngine::doStopSound(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    destroySound(slot);
+}
+
+void AccessibleAudioEngine::doStopAllSounds(SoundAction& action) {
+    auto it = sounds.find(action.handle);
+    if (it == sounds.end())
+        return;
+    SoundSlots& slots = it->second;
+    for (int i = 0; i < AAE_SLOTS_PER_HANDLE; i++) {
+        if (slots[i].active)
+            destroySound(&slots[i]);
+    }
+    sounds.erase(it);
+}
+
+void AccessibleAudioEngine::doSetPitch(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    slot->extras.pitch = action.pitch;
+    float pitch = action.pitch;
+    if (slot->extras.z < 0)
+        pitch *= (1.0 - slot->extras.pitchBehindModifier);
+    ma_sound_set_pitch(&slot->sound, pitch);
+}
+
+void AccessibleAudioEngine::doSetPitchBehindModifier(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    slot->extras.pitchBehindModifier = action.pitch;
+}
+
+void AccessibleAudioEngine::doSetVolume(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    ma_sound_set_volume(&slot->sound, action.pitch);
+}
+
+void AccessibleAudioEngine::doSetPan(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    ma_sound_set_pan(&slot->sound, action.pan);
+}
+
+void AccessibleAudioEngine::doSetFilter(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    slot->extras.cutoff = action.cutoff;
+    ma_lpf_config config = ma_lpf_config_init(ma_format_f32, AAE_CHANNELS, AAE_SAMPLE_RATE,
+                                              lerp_aae(0.0, AAE_SAMPLE_RATE / 2, action.cutoff), AAE_LPF_ORDER);
+    ma_lpf_reinit(&config, &slot->extras.filter);
+}
+
+void AccessibleAudioEngine::doSeekSound(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    ma_sound_seek_to_pcm_frame(&slot->sound, action.offset);
+}
+
+void AccessibleAudioEngine::doSetSoundPos(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    slot->extras.x = action.posX;
+    slot->extras.y = action.posY;
+    slot->extras.z = action.posZ;
+    slot->extras.distToPlayer = action.distToPlayer;
+    slot->extras.maxDistance = action.maxDistance;
+    float pitch = slot->extras.pitch;
+    if (action.posZ < 0)
+        pitch *= (1.0 - slot->extras.pitchBehindModifier);
+    ma_sound_set_pitch(&slot->sound, pitch);
+}
+void AccessibleAudioEngine::garbageCollect() {
+    for (auto i = sounds.begin(); i != sounds.end();) {
+        bool deadSlots = true;
+        for (int x = 0; x < AAE_SLOTS_PER_HANDLE; x++) {
+            if (i->second[x].active) {
+                if (!ma_sound_is_playing(&i->second[x].sound)) {
+                    destroySound(&i->second[x]);
+                } else {
+                    deadSlots = false;
+                }
+            }
+        }
+        if (deadSlots) {
+            i = sounds.erase(i);
+        } else {
+            i++;
+        }
+    }
+    framesUntilGC = AAE_GC_INTERVAL;
+}
+
+void AccessibleAudioEngine::processAudioJobs() {
+    ma_job job;
+    while (ma_resource_manager_next_job(&resourceManager, &job) == MA_SUCCESS)
+        ma_job_process(&job);
+}
+
+bool AccessibleAudioEngine::initSoundExtras(SoundSlot* slot) {
+    ma_node_config config = ma_node_config_init();
+    config.inputBusCount = 1;
+    config.outputBusCount = 1;
+    config.pInputChannels = positioner_channels;
+    config.pOutputChannels = positioner_channels;
+    config.vtable = &positioner_vtable;
+    memset(&slot->extras, 0, sizeof(SoundExtras));
+    if (ma_node_init(&engine.nodeGraph, &config, NULL, &slot->extras) != MA_SUCCESS)
+        return false;
+    ma_panner_config pc = ma_panner_config_init(ma_format_f32, AAE_CHANNELS);
+    pc.mode = ma_pan_mode_balance;
+    ma_panner_init(&pc, &slot->extras.panner);
+    ma_gainer_config gc = ma_gainer_config_init(
+        AAE_CHANNELS,
+        AAE_SAMPLE_RATE / 20); // Allow one in-game frame for the gain to work its way towards the target value.
+    if (ma_gainer_init(&gc, NULL, &slot->extras.gainer) != MA_SUCCESS)
+        return false;
+    ma_lpf_config fc =
+        ma_lpf_config_init(ma_format_f32, AAE_CHANNELS, AAE_SAMPLE_RATE, AAE_SAMPLE_RATE / 2, AAE_LPF_ORDER);
+    ma_lpf_init(&fc, NULL, &slot->extras.filter);
+    slot->extras.cutoff = 1.0f;
+    slot->extras.pitch = 1.0f;
+    slot->extras.pitchBehindModifier = 0.0f;
+    ma_node_attach_output_bus(&slot->sound, 0, &slot->extras, 0);
+    return true;
+}
+
+void AccessibleAudioEngine::destroySound(SoundSlot* slot) {
+    if (slot->extras.base.pNodeGraph != NULL) {
+        ma_node_detach_all_output_buses(&slot->extras);
+        ma_sound_uninit(&slot->sound);
+        ma_gainer_uninit(&slot->extras.gainer, NULL);
+    } else {
+        ma_sound_uninit(&slot->sound);
+    }
+
+    slot->active = false;
+}
+
+AccessibleAudioEngine::AccessibleAudioEngine() {
+    initialized = 0;
+    ma_resource_manager_config rmc = ma_resource_manager_config_init();
+    rmc.decodedChannels = AAE_CHANNELS;
+    rmc.decodedFormat = ma_format_f32;
+    rmc.decodedSampleRate = AAE_SAMPLE_RATE;
+    rmc.flags = MA_RESOURCE_MANAGER_FLAG_NON_BLOCKING;
+    rmc.jobThreadCount = 0;
+    if (ma_resource_manager_init(&rmc, &resourceManager) != MA_SUCCESS)
+        destroyAndThrow("AccessibleAudioEngine: Unable to initialize the resource manager.");
+    initialized = 1;
+    if (ma_pcm_rb_init(ma_format_f32, AAE_CHANNELS, AAE_MAX_BUFFER_SIZE, NULL, NULL, &preparedOutput) != MA_SUCCESS)
+        destroyAndThrow("AccessibleAudioEngine: Unable to initialize the output buffer.");
+    initialized = 2;
+    ma_engine_config ec = ma_engine_config_init();
+    ec.channels = AAE_CHANNELS;
+    ec.noDevice = true;
+    ec.sampleRate = AAE_SAMPLE_RATE;
+    ec.pResourceManager = &resourceManager;
+    ec.listenerCount = 1;
+
+    if (ma_engine_init(&ec, &engine) != MA_SUCCESS)
+        destroyAndThrow("AccessibleAudioEngine: Unable to initialize the audio engine.");
+    initialized = 3;
+    nextOutgoingSoundAction = 0;
+    framesUntilGC = AAE_GC_INTERVAL;
+    thread = std::thread(&AccessibleAudioEngine::runThread, this);
+}
+
+AccessibleAudioEngine::~AccessibleAudioEngine() {
+    // Place a terminate command on the top of the pile, then wait for thread to die.
+    SoundAction action;
+    action.command = AAE_TERMINATE;
+    postHighPrioritySoundAction(action);
+    thread.join();
+    destroy();
+}
+
+void AccessibleAudioEngine::mix(int16_t* ogBuffer, uint32_t nFrames) {
+    float sourceChunk[AAE_MIX_CHUNK_SIZE * AAE_CHANNELS];
+    float mixedChunk[AAE_MIX_CHUNK_SIZE * AAE_CHANNELS];
+    while (nFrames > 0) {
+        uint32_t nextChunk = std::min<uint32_t>(AAE_MIX_CHUNK_SIZE, nFrames);
+        // This is so that it doesn't matter if we have less output available than expected.
+        ma_silence_pcm_frames(sourceChunk, nextChunk, ma_format_f32, AAE_CHANNELS);
+        ma_silence_pcm_frames(mixedChunk, nextChunk, ma_format_f32, AAE_CHANNELS);
+        retrieve(sourceChunk, nextChunk);
+        // The game's output is changed to 32-bit floating point samples.
+        ma_pcm_s16_to_f32(mixedChunk, ogBuffer, nextChunk * AAE_CHANNELS, ma_dither_mode_none);
+        ma_mix_pcm_frames_f32(mixedChunk, sourceChunk, nextChunk, AAE_CHANNELS, 1.0);
+        // If we've gone over 1.0, we'll need to scale back before we go back to 16-bit or we'll distort.
+        float scalar = 1.0;
+        for (uint32_t i = 0; i < nextChunk * AAE_CHANNELS; i++)
+            scalar = std::max(scalar, mixedChunk[i]);
+        if (scalar > 1.0) {
+            scalar = 1.0 / scalar;
+            for (uint32_t i = 0; i < nextChunk * AAE_CHANNELS; i++)
+                mixedChunk[i] *= scalar;
+        }
+        // Chunk is ready to go out via the game's usual channels
+        ma_pcm_f32_to_s16(ogBuffer, mixedChunk, nextChunk * AAE_CHANNELS, ma_dither_mode_triangle);
+        ogBuffer += nextChunk * AAE_CHANNELS;
+        nFrames -= nextChunk;
+    }
+}
+
+void AccessibleAudioEngine::playSound(uintptr_t handle, uint8_t slot, const char* path) {
+    if (slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.handle = handle;
+    action.slot = slot;
+    action.command = AAE_START;
+    action.path = path;
+}
+
+void AccessibleAudioEngine::stopSound(uintptr_t handle, uint8_t slot) {
+    if (slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_STOP;
+    action.handle = (uintptr_t)handle;
+    action.slot = slot;
+}
+
+void AccessibleAudioEngine::stopAllSounds(uintptr_t handle) {
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_STOP_ALL;
+    action.handle = handle;
+}
+
+void AccessibleAudioEngine::setPitch(uintptr_t handle, uint8_t slot, float pitch) {
+    if (slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_PITCH;
+    action.handle = handle;
+    action.slot = slot;
+    action.pitch = pitch;
+}
+
+void AccessibleAudioEngine::setPitchBehindModifier(uintptr_t handle, uint8_t slot, float mod) {
+    if (slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_PITCH_BEHIND;
+    action.handle = handle;
+    action.slot = slot;
+    action.pitch = mod;
+}
+
+void AccessibleAudioEngine::setVolume(uintptr_t handle, uint8_t slot, float volume) {
+    if (slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_VOLUME;
+    action.handle = handle;
+    action.slot = slot;
+    action.volume = volume;
+}
+
+void AccessibleAudioEngine::setPan(uintptr_t handle, uint8_t slot, float pan) {
+    if (slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_PAN;
+    action.handle = handle;
+    action.slot = slot;
+    action.pan = pan;
+}
+void AccessibleAudioEngine::setFilter(uintptr_t handle, uint8_t slot, float cutoff) {
+    if (slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    if (cutoff < 0.0 || cutoff > 1.0)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.handle = handle;
+    action.slot = slot;
+    action.command = AAE_FILTER;
+    action.cutoff = cutoff;
+}
+
+void AccessibleAudioEngine::seekSound(uintptr_t handle, uint8_t slot, size_t offset) {
+    if (slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.handle = handle;
+    action.slot = slot;
+    action.command = AAE_SEEK;
+    action.offset = offset;
+}
+
+void AccessibleAudioEngine::setSoundPosition(uintptr_t handle, uint8_t slot, float posX, float posY, float posZ,
+                                             float distToPlayer, float maxDistance) {
+    if (slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_POS;
+    action.handle = handle;
+    action.slot = slot;
+    action.posX = posX;
+    action.posY = posY;
+    action.posZ = posZ;
+    action.distToPlayer = distToPlayer;
+    action.maxDistance = maxDistance;
+}
+
+void AccessibleAudioEngine::prepare() {
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_PREPARE;
+    // This is called once at the end of every frame, so now is the time to post all of the accumulated commands.
+    postSoundActions();
+}

soh/soh/Enhancements/accessible-actors/AccessibleAudioEngine.h

@@ -0,0 +1,147 @@
+#pragma once
+#include <stdint.h>
+#include <thread>
+#include <mutex>
+#include <deque>
+#include <condition_variable>
+#include <string>
+#include <unordered_map>
+#include <array>
+
+#include "soh/Enhancements/audio/miniaudio.h"
+
+#define AAE_SOUND_ACTION_BATCH_SIZE 64
+#define AAE_SLOTS_PER_HANDLE 8
+
+struct SoundAction {
+    uintptr_t handle; // This handle is user-defined and uniquely identifies a sound source. It can be anything, but the
+                      // address of an object with which the sound is associated is recommended.
+    uint8_t slot;     // Allows multiple sounds per handle. The exact number is controlled by AAE_SOUNDS_PER_HANDLE.
+    uint8_t command;  // One of the items belonging to AAE_COMMANDS.
+    std::string path; // If command is AAE_START, this is the path to the desired resource.
+    union {
+        float pitch;
+        float volume;
+        float pan;
+        float cutoff;
+        size_t offset; // for seeking.
+        float distance;
+    };
+
+    // Position and rotation vectors for AAE_POS
+    float posX;
+    float posY;
+    float posZ;
+    float distToPlayer;
+    float maxDistance;
+    uint32_t frames; // If command is AAE_PREPARE, this tells the engine how many PCM frames to get ready.
+};
+
+typedef struct {
+    ma_node_base base;
+    ma_panner panner;
+    ma_gainer gainer;
+    ma_lpf filter;
+    float cutoff;
+    float x;
+    float y;
+    float z;
+    float distToPlayer;
+    float maxDistance;
+    float pitch;
+    float pitchBehindModifier;
+} SoundExtras; // Used for attenuation and other effects.
+
+typedef struct {
+    ma_sound sound;
+    SoundExtras extras;
+
+    bool active;
+} SoundSlot;
+typedef std::array<SoundSlot, AAE_SLOTS_PER_HANDLE> SoundSlots;
+
+class AccessibleAudioEngine {
+    int initialized;
+    ma_engine engine;
+    ma_pcm_rb preparedOutput;             // Lock-free single producer single consumer.
+    std::deque<SoundAction> soundActions; // A command cue.
+    std::thread thread;
+    std::condition_variable cv;
+    std::mutex mtx;
+    std::unordered_map<uintptr_t, SoundSlots> sounds;
+    SoundAction
+        outgoingSoundActions[AAE_SOUND_ACTION_BATCH_SIZE]; // Allows batch delivery of SoundActions to the FIFO to
+                                                           // minimize the amount of time spent locking and unlocking.
+    int nextOutgoingSoundAction;
+    int framesUntilGC;
+    void destroy(); // Called by the destructor, or if a throw occurs during construction.
+    // Dismantal a partial initialization and throw an exception.
+    void destroyAndThrow(const char* exceptionText);
+
+    // Retrieve some audio from the output buffer. This is to be performed by the audio thread. May return less data
+    // than requested.
+    uint32_t retrieve(float* buffer, uint32_t nFrames);
+    // Functions dealing with the command queue.
+    int getSoundActions(SoundAction* dest, int limit);
+    void postSoundActions();
+    void postHighPrioritySoundAction(SoundAction& action); // For now this is used only for termination events.
+
+    SoundAction& getNextOutgoingSoundAction();
+    void runThread();
+    // Find a sound by handle and slot, if it exists.
+    SoundSlot* findSound(SoundAction& action);
+
+    // Functions which correspond to SoundAction commands.
+    // Ready a sound for playback.
+    void doPlaySound(SoundAction& action);
+    void doStopSound(SoundAction& action);
+    void doStopAllSounds(SoundAction& action);
+
+    void doSetPitch(SoundAction& action);
+    void doSetPitchBehindModifier(SoundAction& action);
+    void doSetVolume(SoundAction& action);
+    void doSetPan(SoundAction& action);
+    void doSetFilter(SoundAction& action);
+    void doSeekSound(SoundAction& action);
+
+    void doSetListenerPos(SoundAction& action);
+    void doSetSoundPos(SoundAction& action);
+    // Generate some output, and store it in the output buffer for later retrieval. May generate less output than
+    // requested if buffer space is insufficient.
+    void doPrepare(SoundAction& action);
+    // Run every so often to clean up expired sound handles.
+    void garbageCollect();
+    // Run MiniAudio's jobs.
+    void processAudioJobs();
+    // Set up the panner and other effect processing on a sound slot.
+    bool initSoundExtras(SoundSlot* slot);
+    void destroySound(SoundSlot* slot);
+
+  public:
+    AccessibleAudioEngine();
+    ~AccessibleAudioEngine();
+    // Mix the game's audio with this engine's audio to produce the final mix. To be performed exclusively in the audio
+    // thread. Mixing is done in-place (meaning the buffer containing the game's audio is overwritten with the mixed
+    // content).
+    void mix(int16_t* ogBuffer, uint32_t nFrames);
+    // Start playing a sound.
+    void playSound(uintptr_t handle, uint8_t slot, const char* path);
+    void stopSound(uintptr_t handle, uint8_t slot);
+    // Stop all sounds belonging to a handle.
+    void stopAllSounds(uintptr_t handle);
+
+    void setPitch(uintptr_t handle, uint8_t slot, float pitch);
+    void setPitchBehindModifier(uintptr_t handle, uint8_t slot, float mod);
+    void setVolume(uintptr_t handle, uint8_t slot, float volume);
+    void setPan(uintptr_t handle, uint8_t slot, float pan);
+    // Set the lowpass filter cutoff. Set to 1.0 for no filtering.
+    void setFilter(uintptr_t handle, uint8_t slot, float cutoff);
+    // Seek the sound to a particular PCM frame.
+    void seekSound(uintptr_t handle, uint8_t slot, size_t offset);
+    void setSoundPosition(uintptr_t handle, uint8_t slot, float posX, float posY, float posZ, float distToPlayer,
+                          float maxDistance);
+    // Schedule the preparation of output for delivery.
+    void prepare();
+
+    ma_resource_manager resourceManager;
+};

soh/soh/Enhancements/accessible-actors/ActorAccessibility.cpp

@@ -0,0 +1,823 @@
+#include "ActorAccessibility.h"
+#include "AccessibleAudioEngine.h"
+#include "soh/OTRGlobals.h"
+#include "resource/type/Blob.h"
+
+#include <map>
+#include <random>
+#include <vector>
+#include <functions.h>
+#include <variables.h>
+#include <macros.h>
+#include "ResourceType.h"
+#include "SfxExtractor.h"
+
+#include <sstream>
+#include "File.h"
+#include <unordered_set>
+#include "soh/Enhancements/speechsynthesizer/SpeechSynthesizer.h"
+#include "soh/Enhancements/tts/tts.h"
+#include "soh/Enhancements/game-interactor/GameInteractor.h"
+
+extern "C" {
+extern PlayState* gPlayState;
+extern bool freezeGame;
+extern bool freezeActors;
+}
+
+const char* GetLanguageCode();
+
+// This is the amount in DB that a sound will be reduced by when it is at the maximum distance from the player.
+#define MAX_DB_REDUCTION 35
+
+extern "C" {
+// Used to tell where polygons are located.
+void CollisionPoly_GetVertices(CollisionPoly* poly, Vec3s* vtxList, Vec3f* dest);
+}
+
+typedef struct {
+    union {
+        struct {
+            s16 sceneIndex; // Corresponds directly to the game's scene indices.
+            s16 roomIndex;  // Corresponds directly to the game's room indices.
+        } values;
+        s32 raw; // Combination of the two which can be used for dictionary lookups.
+    };
+} SceneAndRoom;
+
+// Maps actors to their accessibility policies, which describe how accessibility should treat them.
+typedef std::map<s16, ActorAccessibilityPolicy> SupportedActors_t;
+typedef std::map<Actor*, uint64_t>
+    TrackedActors_t; // Maps real actors to internal IDs specific to accessibility.
+                     // Maps internal IDs to wrapped actor objects. These actors can be real or virtual.
+typedef std::map<uint64_t, AccessibleActor> AccessibleActorList_t;
+typedef std::vector<AccessibleActor> VAList_t; // Denotes a list of virtual actors specific to a single room.
+typedef std::map<s32, VAList_t> VAZones_t; // Maps room/scene indices to their corresponding virtual actor collections.
+// A list of scenes which have already been visited (since the game was launched). Used to prevent
+// re-creation of terrain VAs every time the player reloads a scene.
+typedef std::unordered_set<s16> SceneList_t;
+
+struct SfxRecord {
+    std::string path;
+    std::shared_ptr<Ship::File> resource;
+};
+
+class AudioGlossaryData {
+  public:
+    AccessibleActorList_t accessibleActorList;
+    AccessibleActorList_t::iterator current = accessibleActorList.begin();
+    bool GlossaryStarted = false;
+    u16 frameCount = 0;
+    s16 currentScene = -1;
+    s8 currentRoom = -1;
+    s8 cooldown = 0;
+};
+
+class ActorAccessibility {
+  public:
+    bool isOn = false;
+    uint64_t nextActorID = 0;
+    SupportedActors_t supportedActors;
+    TrackedActors_t trackedActors;
+    AccessibleActorList_t accessibleActorList;
+    AudioGlossaryData* glossary;
+    VAZones_t vaZones;
+    SceneList_t sceneList;
+    AccessibleAudioEngine* audioEngine;
+    SfxExtractor sfxExtractor;
+    // Maps internal sfx to external (prerendered) resources
+    std::unordered_map<s16, SfxRecord> sfxMap;
+    s16 currentScene = -1;
+    s8 currentRoom = -1;
+    bool currentRoomClear = false;
+    u8 framesUntilChime = 0;
+    Vec3f prevPos = { 0, 0, 0 };
+    s16 prevYaw = 0;
+    bool extractSfx = false;
+    TerrainCueState* terrainCues = nullptr;
+    VirtualActorList* currentSceneGlobal = nullptr;
+    VirtualActorList* currentRoomLocal = nullptr;
+};
+static ActorAccessibility* aa;
+
+uint64_t ActorAccessibility_GetNextID() {
+    return aa->nextActorID++;
+}
+
+void ActorAccessibility_PrepareNextAudioFrame();
+
+// Hooks for game-interactor.
+void ActorAccessibility_OnActorInit(void* actor) {
+    ActorAccessibility_TrackNewActor((Actor*)actor);
+}
+void ActorAccessibility_OnGameFrameUpdate() {
+    if (gPlayState == NULL)
+        return;
+    if (!GameInteractor::IsSaveLoaded() && !aa->extractSfx)
+        return; // Title screen, skip.
+
+    ActorAccessibility_RunAccessibilityForAllActors(gPlayState);
+}
+
+void ActorAccessibility_OnActorDestroy(void* actor) {
+    ActorAccessibility_RemoveTrackedActor((Actor*)actor);
+}
+
+void ActorAccessibility_OnGameStillFrozen() {
+    if (gPlayState == NULL)
+        return;
+    if (aa->extractSfx)
+        ActorAccessibility_HandleSoundExtractionMode(gPlayState);
+}
+
+void ActorAccessibility_Init() {
+    aa = new ActorAccessibility();
+    aa->glossary = new AudioGlossaryData();
+    aa->isOn = CVarGetInteger(CVAR_SETTING("A11yAudioInteraction"), 0);
+    if (!aa->isOn)
+        return;
+    aa->extractSfx = !std::filesystem::exists(Ship::Context::GetPathRelativeToAppBundle("accessibility.o2r"));
+    if (aa->extractSfx)
+        freezeGame = true;
+    ActorAccessibility_InitAudio();
+    ActorAccessibility_InitActors();
+    GameInteractor::Instance->RegisterGameHook<GameInteractor::OnActorInit>(ActorAccessibility_OnActorInit);
+    GameInteractor::Instance->RegisterGameHook<GameInteractor::OnActorDestroy>(ActorAccessibility_OnActorDestroy);
+
+    GameInteractor::Instance->RegisterGameHook<GameInteractor::OnPlayerUpdate>(ActorAccessibility_OnGameFrameUpdate);
+    GameInteractor::Instance->RegisterGameHook<GameInteractor::OnGameStillFrozen>(ActorAccessibility_OnGameStillFrozen);
+}
+
+void ActorAccessibility_Shutdown() {
+    ActorAccessibility_ShutdownAudio();
+    delete aa;
+}
+
+void ActorAccessibility_InitPolicy(ActorAccessibilityPolicy* policy, const char* englishName) {
+    policy->distance = 500;
+    policy->ydist = 80;
+    policy->englishName = englishName;
+    policy->n = 20;
+    policy->pitch = 1.5;
+    policy->runsAlways = false;
+    policy->volume = 1.0;
+    policy->pitchModifier = 0.1;
+    policy->aimAssist.isProvider = 0;
+    policy->aimAssist.sfx = NA_SE_SY_HITPOINT_ALARM;
+    policy->aimAssist.tolerance = 0.0;
+}
+
+void ActorAccessibility_InitPolicy(ActorAccessibilityPolicy* policy, const char* englishName,
+                                   ActorAccessibilityCallback callback) {
+    policy->callback = callback;
+    policy->sound = 0;
+    ActorAccessibility_InitPolicy(policy, englishName);
+}
+
+void ActorAccessibility_InitPolicy(ActorAccessibilityPolicy* policy, const char* englishName, s16 sfx) {
+    policy->callback = nullptr;
+    policy->sound = sfx;
+    ActorAccessibility_InitPolicy(policy, englishName);
+}
+
+void ActorAccessibility_AddSupportedActor(s16 type, ActorAccessibilityPolicy policy) {
+    aa->supportedActors[type] = policy;
+}
+
+void ActorAccessibility_AddTerrainCues(AccessibleActor* actor) {
+    aa->terrainCues = InitTerrainCueState(actor);
+}
+
+ActorAccessibilityPolicy* ActorAccessibility_GetPolicyForActor(s16 type) {
+    SupportedActors_t::iterator i = aa->supportedActors.find(type);
+    if (i == aa->supportedActors.end())
+        return NULL;
+    return &i->second;
+}
+
+int ActorAccessibility_GetRandomStartingFrameCount(int min, int max) {
+    static std::mt19937 gen;
+    std::uniform_int_distribution<> dist(min, max);
+    return dist(gen);
+}
+
+void ActorAccessibility_TrackNewActor(Actor* actor) {
+    // Don't track actors for which no accessibility policy has been configured.
+    ActorAccessibilityPolicy* policy = ActorAccessibility_GetPolicyForActor(actor->id);
+    if (policy == NULL)
+        return;
+    AccessibleActor accessibleActor;
+    accessibleActor.instanceID = ActorAccessibility_GetNextID();
+    accessibleActor.actor = actor;
+    accessibleActor.id = actor->id;
+    // Stagger the start times so that all of the sounds don't play at exactly the same time.
+    accessibleActor.frameCount = ActorAccessibility_GetRandomStartingFrameCount(0, policy->n);
+    accessibleActor.basePitch = policy->pitch;
+    accessibleActor.policy = *policy;
+    accessibleActor.currentPitch = accessibleActor.policy.pitch;
+    accessibleActor.baseVolume = accessibleActor.policy.volume;
+    accessibleActor.currentVolume = accessibleActor.policy.volume;
+    accessibleActor.sceneIndex = 0;
+    accessibleActor.managedSoundSlots = 0;
+    accessibleActor.aimFramesSinceAimAssist = 255;
+    accessibleActor.aimFrequency = 10;
+
+    aa->trackedActors[actor] = accessibleActor.instanceID;
+    aa->accessibleActorList[accessibleActor.instanceID] = accessibleActor;
+}
+
+void ActorAccessibility_RemoveTrackedActor(Actor* actor) {
+    TrackedActors_t::iterator i = aa->trackedActors.find(actor);
+    if (i == aa->trackedActors.end())
+        return;
+    uint64_t id = i->second;
+    aa->trackedActors.erase(i);
+    AccessibleActorList_t::iterator i2 = aa->accessibleActorList.find(id);
+    if (i2 == aa->accessibleActorList.end())
+        return;
+    ActorAccessibility_StopAllSoundsForActor(&i2->second);
+    aa->accessibleActorList.erase(i2);
+}
+
+f32 ActorAccessibility_DBToLinear(float gain) {
+    return powf(10.0, gain / 20.0f);
+}
+
+f32 ActorAccessibility_ComputeCurrentVolume(f32 maxDistance, f32 xzDistToPlayer) {
+    if (maxDistance == 0)
+        return 0.0;
+    f32 absDistance = fabs(xzDistToPlayer);
+    f32 db = LERP(0.0 - MAX_DB_REDUCTION, 0.0, (maxDistance - absDistance) / maxDistance);
+
+    return ActorAccessibility_DBToLinear(db);
+}
+
+const char* ActorAccessibility_MapSfxToExternalAudio(s16 sfxId);
+void ActorAccessibility_PlaySound(void* handle, int slot, s16 sfxId) {
+    const char* path = ActorAccessibility_MapSfxToExternalAudio(sfxId);
+    if (path == NULL)
+        return;
+    aa->audioEngine->playSound((uintptr_t)handle, slot, path);
+}
+
+void ActorAccessibility_StopSound(void* handle, int slot) {
+    aa->audioEngine->stopSound((uintptr_t)handle, slot);
+}
+void ActorAccessibility_StopAllSounds(void* handle) {
+    aa->audioEngine->stopAllSounds((uintptr_t)handle);
+}
+void ActorAccessibility_SetSoundPitch(void* handle, int slot, float pitch) {
+    aa->audioEngine->setPitch((uintptr_t)handle, slot, pitch);
+}
+void ActorAccessibility_SetPitchBehindModifier(void* handle, int slot, float mod) {
+    aa->audioEngine->setPitchBehindModifier((uintptr_t)handle, slot, mod);
+}
+void ActorAccessibility_SetSoundPos(void* handle, int slot, Vec3f* pos, f32 distToPlayer, f32 maxDistance) {
+    aa->audioEngine->setSoundPosition((uintptr_t)handle, slot, pos->x, pos->y, pos->z, distToPlayer, maxDistance);
+}
+void ActorAccessibility_SetSoundVolume(void* handle, int slot, float volume) {
+    aa->audioEngine->setVolume((uintptr_t)handle, slot, volume);
+}
+void ActorAccessibility_SetSoundPan(void* handle, int slot, float pan) {
+    aa->audioEngine->setPan((uintptr_t)handle, slot, pan);
+}
+void ActorAccessibility_SetSoundFilter(void* handle, int slot, float cutoff) {
+    aa->audioEngine->setFilter((uintptr_t)handle, slot, cutoff);
+}
+void ActorAccessibility_SeekSound(void* handle, int slot, size_t offset) {
+    aa->audioEngine->seekSound((uintptr_t)handle, slot, offset);
+}
+
+void ActorAccessibility_ConfigureSoundForActor(AccessibleActor* actor, int slot) {
+    ActorAccessibility_SetSoundPitch(actor, slot, actor->policy.pitch);
+    ActorAccessibility_SetPitchBehindModifier(actor, slot, actor->policy.pitchModifier);
+    ActorAccessibility_SetSoundPos(actor, slot, &actor->projectedPos, actor->xyzDistToPlayer, actor->policy.distance);
+    ActorAccessibility_SetSoundVolume(actor, slot, actor->policy.volume);
+    actor->managedSoundSlots |= 1 << slot;
+}
+
+void ActorAccessibility_PlaySoundForActor(AccessibleActor* actor, int slot, s16 sfxId) {
+    if (slot < 0 || slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    ActorAccessibility_PlaySound(actor, slot, sfxId);
+    ActorAccessibility_ConfigureSoundForActor(actor, slot);
+}
+
+void ActorAccessibility_StopSoundForActor(AccessibleActor* actor, int slot) {
+    if (slot < 0 || slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    ActorAccessibility_StopSound(actor, slot);
+    actor->managedSoundSlots &= ~(1 << slot);
+}
+
+void ActorAccessibility_StopAllSoundsForActor(AccessibleActor* actor) {
+    ActorAccessibility_StopAllSounds(actor);
+    actor->managedSoundSlots = 0;
+}
+
+void ActorAccessibility_CopyParamsFromRealActor(AccessibleActor* actor) {
+    Player* player = GET_PLAYER(actor->play);
+    if (actor->actor == NULL)
+        return;
+    actor->projectedPos = actor->actor->projectedPos;
+    actor->xzDistToPlayer = actor->actor->xzDistToPlayer;
+    actor->isDrawn = actor->actor->isDrawn;
+    actor->pos = actor->actor->world.pos;
+    actor->xyzDistToPlayer = sqrtf(actor->actor->xyzDistToPlayerSq);
+}
+
+void ActorAccessibility_StopAllVirtualActors(VirtualActorList* list) {
+    if (list == NULL)
+        return;
+
+    VAList_t* val = (VAList_t*)list;
+    for (auto i = val->begin(); i != val->end(); i++)
+        ActorAccessibility_StopAllSounds((void*)&(*i));
+}
+
+void ActorAccessibility_RunAccessibilityForActor(PlayState* play, AccessibleActor* actor) {
+    actor->play = play;
+    if (actor->actor != nullptr) {
+        ActorAccessibility_CopyParamsFromRealActor(actor);
+    } else {
+        Player* player = GET_PLAYER(play);
+        f32 w = 0.0f;
+        // Set actor->projectedPos.
+        SkinMatrix_Vec3fMtxFMultXYZW(&play->viewProjectionMtxF, &actor->pos, &actor->projectedPos, &w);
+        actor->xzDistToPlayer = Math_Vec3f_DistXZ(&actor->pos, &player->actor.world.pos);
+        actor->xyzDistToPlayer = Math_Vec3f_DistXYZ(&actor->pos, &player->actor.world.pos);
+        actor->yDistToPlayer = fabs((actor->pos.y) - (player->actor.world.pos.y));
+    }
+
+    if (actor->actor != NULL && fabs(actor->actor->yDistToPlayer) > actor->policy.ydist) {
+        return;
+    }
+    // Send sound parameters to the new audio engine. Eventually remove the old stuff once all actors are carried over.
+    for (int i = 0; i < AAE_SLOTS_PER_HANDLE; i++) {
+        if (actor->managedSoundSlots & (1 << i)) {
+            ActorAccessibility_SetSoundPos(actor, i, &actor->projectedPos, actor->xyzDistToPlayer,
+                                           actor->policy.distance);
+            // Judgement call: pitch changes are rare enough that it doesn't make sense to pay the cost of updating it
+            // every frame. If you want a pitch change, call the function as needed.
+        }
+    }
+    actor->frameCount++;
+    if (aa->glossary->GlossaryStarted) {
+        aa->glossary->frameCount++;
+    }
+    if (!actor->policy.runsAlways && actor->xyzDistToPlayer > actor->policy.distance) {
+        return;
+    } else if (actor->isDrawn == 0 && actor->actor->id != ACTOR_EN_IT && actor->actor->id != ACTOR_EN_OKARINA_TAG &&
+               !aa->glossary->GlossaryStarted) {
+        return;
+    }
+
+    if (actor->policy.aimAssist.isProvider) {
+        Player* player = GET_PLAYER(play);
+        if ((player->stateFlags1 & PLAYER_STATE1_FIRST_PERSON) &&
+            ((actor->policy.aimAssist.isProvider & AIM_CUP) ||
+             (player->stateFlags1 & (PLAYER_STATE1_USING_BOOMERANG | PLAYER_STATE1_ITEM_IN_HAND)))) {
+            bool aim = false;
+            switch (player->heldItemAction) {
+                case PLAYER_IA_BOW:
+                case PLAYER_IA_BOW_FIRE:
+                case PLAYER_IA_BOW_ICE:
+                case PLAYER_IA_BOW_LIGHT:
+                case PLAYER_IA_BOW_0C:
+                case PLAYER_IA_BOW_0D:
+                case PLAYER_IA_BOW_0E:
+                    aim = actor->policy.aimAssist.isProvider & AIM_BOW;
+                    break;
+                case PLAYER_IA_SLINGSHOT:
+                    aim = actor->policy.aimAssist.isProvider & AIM_SLING;
+                    break;
+                case PLAYER_IA_HOOKSHOT:
+                case PLAYER_IA_LONGSHOT:
+                    aim = actor->policy.aimAssist.isProvider & AIM_HOOK;
+                    break;
+                case PLAYER_IA_BOOMERANG:
+                    aim = actor->policy.aimAssist.isProvider & AIM_BOOM;
+                    break;
+                case PLAYER_IA_NONE:
+                    aim = actor->policy.aimAssist.isProvider & AIM_CUP;
+                    break;
+            }
+            if (aim) {
+                auto aimAssistProps = ActorAccessibility_ProvideAimAssistForActor(actor);
+                if (++actor->aimFramesSinceAimAssist >= actor->aimFrequency) {
+                    actor->aimFramesSinceAimAssist = 0;
+                    ActorAccessibility_PlaySoundForActor(actor, 7, actor->policy.aimAssist.sfx);
+                }
+                ActorAccessibility_SetSoundPitch(actor, 7, aimAssistProps.pitch);
+                ActorAccessibility_SetSoundVolume(actor, 7, aimAssistProps.volume);
+                ActorAccessibility_SetSoundPan(actor, 7, aimAssistProps.pan);
+            }
+        } else {
+            // Make sure there's no delay the next time you draw your bow or whatever.
+            actor->aimFramesSinceAimAssist = 255;
+        }
+    }
+
+    if (actor->frameCount % actor->policy.n == 0) {
+        if (actor->policy.callback != nullptr) {
+            actor->policy.callback(actor);
+        } else if (actor->policy.sound != 0) {
+            ActorAccessibility_PlaySoundForActor(actor, 0, actor->policy.sound);
+        }
+    }
+}
+
+void ActorAccessibility_RunAccessibilityForAllActors(PlayState* play) {
+    Player* player = GET_PLAYER(play);
+
+    if (aa->currentScene != play->sceneNum) {
+        if (aa->terrainCues)
+            ActorAccessibility_StopAllSounds(aa->terrainCues);
+        ActorAccessibility_StopAllVirtualActors(aa->currentSceneGlobal);
+        ActorAccessibility_StopAllVirtualActors(aa->currentRoomLocal);
+        ActorAccessibility_InterpretCurrentScene(play);
+        aa->currentSceneGlobal = ActorAccessibility_GetVirtualActorList(play->sceneNum, -1);
+        aa->currentRoomLocal = ActorAccessibility_GetVirtualActorList(play->sceneNum, play->roomCtx.curRoom.num);
+
+        aa->currentScene = play->sceneNum;
+        aa->currentRoom = play->roomCtx.curRoom.num;
+        aa->currentRoomClear = Flags_GetClear(play, aa->currentRoom);
+    } else if (aa->currentRoom != play->roomCtx.curRoom.num) {
+        ActorAccessibility_StopAllVirtualActors(aa->currentRoomLocal);
+        ActorAccessibility_AnnounceRoomNumber(play);
+        aa->currentRoomLocal = ActorAccessibility_GetVirtualActorList(play->sceneNum, play->roomCtx.curRoom.num);
+        aa->currentRoom = play->roomCtx.curRoom.num;
+        aa->currentRoomClear = Flags_GetClear(play, aa->currentRoom);
+    }
+
+    if (aa->glossary->currentScene != play->sceneNum || aa->glossary->currentRoom != play->roomCtx.curRoom.num) {
+        if (aa->glossary->GlossaryStarted) {
+            aa->glossary->cooldown = 0;
+            aa->glossary->GlossaryStarted = false;
+            freezeActors = false;
+        }
+    }
+    if (player->stateFlags1 & PLAYER_STATE1_IN_CUTSCENE) {
+        return;
+    }
+    ActorAccessibility_AudioGlossary(play);
+    if (aa->glossary->GlossaryStarted) {
+        return;
+    }
+
+    ActorAccessibility_GeneralHelper(play);
+
+    // Real actors.
+    for (AccessibleActorList_t::iterator i = aa->accessibleActorList.begin(); i != aa->accessibleActorList.end(); i++)
+        ActorAccessibility_RunAccessibilityForActor(play, &i->second);
+
+    if (aa->terrainCues) {
+        RunTerrainCueState(aa->terrainCues, play);
+    }
+
+    // Virtual actors for the current room and scene.
+    VAList_t* list = (VAList_t*)aa->currentRoomLocal;
+    for (VAList_t::iterator i = list->begin(); i != list->end(); i++)
+        ActorAccessibility_RunAccessibilityForActor(play, &(*i));
+    // Scene-global virtual actors. Most of these are automatically generated VAs from polygons, because there's no way
+    // to sort these into rooms.
+    list = (VAList_t*)aa->currentSceneGlobal;
+    for (VAList_t::iterator i = list->begin(); i != list->end(); i++)
+        ActorAccessibility_RunAccessibilityForActor(play, &(*i));
+
+    // Processes external audio engine.
+    ActorAccessibility_PrepareNextAudioFrame();
+}
+
+void ActorAccessibility_GeneralHelper(PlayState* play) {
+    Player* player = GET_PLAYER(play);
+    if (player == nullptr)
+        return;
+
+    // Report when a room is completed.
+    if (!aa->currentRoomClear && Flags_GetClear(play, aa->currentRoom)) {
+        aa->currentRoomClear = Flags_GetClear(play, aa->currentRoom);
+        ActorAccessibility_AnnounceRoomNumber(play);
+    }
+
+    if (player->actor.wallPoly && player->actor.speedXZ > 0 &&
+        (player->yDistToLedge == 0 || player->yDistToLedge >= 79.0f)) {
+        f32 movedsq = SQ(aa->prevPos.x - player->actor.world.pos.x) + SQ(aa->prevPos.z - player->actor.world.pos.z);
+        if (movedsq < 0.125) {
+            ActorAccessibility_PlaySound(nullptr, 3, NA_SE_IT_WALL_HIT_SOFT);
+            ActorAccessibility_SetSoundVolume(nullptr, 3, 0.5);
+        } else if (movedsq < 9) {
+            ActorAccessibility_PlaySound(nullptr, 3, NA_SE_IT_SHIELD_POSTURE);
+            ActorAccessibility_SetSoundVolume(nullptr, 3, 0.6);
+        } else {
+            ActorAccessibility_PlaySound(nullptr, 3, NA_SE_PL_WALK_WALL);
+            ActorAccessibility_SetSoundVolume(nullptr, 3, std::max(0.3f, 10.0f / movedsq));
+        }
+    }
+
+    bool compassOn = false;
+    if (aa->prevPos.x == player->actor.world.pos.x && aa->prevPos.z == player->actor.world.pos.z) {
+        if (ABS(aa->prevYaw - player->yaw) > 0x400) {
+            compassOn = true;
+            aa->prevYaw = player->yaw;
+        }
+    } else {
+        aa->prevPos = player->actor.world.pos;
+    }
+
+    if (aa->framesUntilChime > 0) {
+        aa->framesUntilChime--;
+    } else {
+        if (!compassOn) {
+            OSContPad* trackerButtonsPressed =
+                std::dynamic_pointer_cast<LUS::ControlDeck>(Ship::Context::GetInstance()->GetControlDeck())->GetPads();
+            compassOn = trackerButtonsPressed != nullptr && (trackerButtonsPressed[0].button & BTN_DDOWN) &&
+                        (trackerButtonsPressed[0].button & BTN_L);
+        }
+        if (compassOn) {
+            ActorAccessibility_PlaySound(nullptr, 0, NA_SE_EV_SHIP_BELL);
+            ActorAccessibility_SetSoundPitch(nullptr, 0, 1.5f + Math_CosS(player->yaw) / 2);
+            ActorAccessibility_SetSoundPan(nullptr, 0, -Math_SinS(player->yaw));
+            s16 range = ABS(((player->yaw + 0xA000) & 0x3FFF) - 0x2000);
+            aa->framesUntilChime = range <= 0x400 ? 10 : range <= 0x1000 ? 20 : 30;
+        }
+    }
+
+    if (fabs(player->unk_860 - 25) < 24.0 && player->heldItemId == 0) {
+        ActorAccessibility_PlaySound(nullptr, 1, NA_SE_SY_WARNING_COUNT_N);
+    }
+
+    if (Player_HoldsHookshot(player) && player->heldActor != NULL && player->actor.scale.y >= 0.0f &&
+        (player->stateFlags1 & PLAYER_STATE1_FIRST_PERSON)) {
+        CollisionPoly* colPoly;
+        s32 bgId;
+        Vec3f firstHit;
+        f32 hookshotLength = ((player->heldItemAction == PLAYER_IA_HOOKSHOT) ? 380.0f : 770.0f) *
+                             CVarGetFloat(CVAR_CHEAT("HookshotReachMultiplier"), 1.0f);
+        Vec3f hookshotEnd = player->heldActor->world.pos;
+        hookshotEnd.x +=
+            Math_SinS(player->heldActor->world.rot.y) * Math_SinS(-player->heldActor->world.rot.x) * hookshotLength;
+        hookshotEnd.y += Math_SinS(-player->heldActor->world.rot.x) * hookshotLength;
+        hookshotEnd.z +=
+            Math_CosS(player->heldActor->world.rot.y) * Math_CosS(-player->heldActor->world.rot.x) * hookshotLength;
+        if (BgCheck_AnyLineTest3(&play->colCtx, &player->heldActor->world.pos, &hookshotEnd, &firstHit, &colPoly, 1, 1,
+                                 1, 1, &bgId)) {
+            if (SurfaceType_IsHookshotSurface(&play->colCtx, colPoly, bgId)) {
+                ActorAccessibility_PlaySound(nullptr, 2, NA_SE_IT_HOOKSHOT_STICK_OBJ);
+                ActorAccessibility_SetSoundVolume(nullptr, 2, 0.5f);
+            }
+        }
+    }
+}
+
+void ActorAccessibility_AudioGlossary(PlayState* play) {
+    if (aa->glossary->GlossaryStarted) {
+        freezeActors = true;
+        AccessibleActor glossaryActor = (*aa->glossary->current).second;
+        ActorAccessibility_CopyParamsFromRealActor(&glossaryActor);
+        glossaryActor.policy.distance = glossaryActor.xzDistToPlayer * 3;
+        glossaryActor.policy.ydist = 1000;
+        glossaryActor.frameCount = aa->glossary->frameCount;
+        ActorAccessibility_RunAccessibilityForActor(play, &glossaryActor);
+    }
+    if (aa->glossary->cooldown != 0) {
+        aa->glossary->cooldown--;
+        return;
+    }
+
+    OSContPad* trackerButtonsPressed =
+        std::dynamic_pointer_cast<LUS::ControlDeck>(Ship::Context::GetInstance()->GetControlDeck())->GetPads();
+    bool comboStartGlossary = trackerButtonsPressed != nullptr && trackerButtonsPressed[0].button & BTN_DUP &&
+                              trackerButtonsPressed[0].button & BTN_L;
+    if (comboStartGlossary) {
+        aa->glossary->GlossaryStarted = true;
+        aa->glossary->current = aa->accessibleActorList.begin();
+        aa->glossary->currentScene = play->sceneNum;
+        aa->glossary->currentRoom = play->roomCtx.curRoom.num;
+        SpeechSynthesizer::Instance->Speak((*aa->glossary->current).second.policy.englishName, GetLanguageCode());
+        return;
+    }
+    bool comboNextGlossary = trackerButtonsPressed != nullptr && trackerButtonsPressed[0].button & BTN_DRIGHT &&
+                             trackerButtonsPressed[0].button & BTN_L;
+    if (comboNextGlossary && aa->glossary->GlossaryStarted) {
+        aa->glossary->current++;
+        if (aa->glossary->current == aa->accessibleActorList.end()) {
+            aa->glossary->current = aa->accessibleActorList.begin();
+        };
+        aa->glossary->cooldown = 5;
+        SpeechSynthesizer::Instance->Speak((*aa->glossary->current).second.policy.englishName, GetLanguageCode());
+    }
+    bool comboPrevGlossary = trackerButtonsPressed != nullptr && trackerButtonsPressed[0].button & BTN_DLEFT &&
+                             trackerButtonsPressed[0].button & BTN_L;
+    if (comboPrevGlossary && aa->glossary->GlossaryStarted) {
+        if (aa->glossary->current != aa->accessibleActorList.begin()) {
+            aa->glossary->current--;
+        }
+        aa->glossary->cooldown = 5;
+
+        SpeechSynthesizer::Instance->Speak((*aa->glossary->current).second.policy.englishName, GetLanguageCode());
+    }
+    bool comboDisableGlossary = trackerButtonsPressed != nullptr && trackerButtonsPressed[0].button & BTN_DDOWN &&
+                                trackerButtonsPressed[0].button & BTN_L;
+    if (comboDisableGlossary) {
+        aa->glossary->cooldown = 0;
+        aa->glossary->GlossaryStarted = false;
+        freezeActors = false;
+    }
+    // Processes external audio engine.
+    ActorAccessibility_PrepareNextAudioFrame();
+}
+
+// Virtual actor config.
+VirtualActorList* ActorAccessibility_GetVirtualActorList(s16 sceneNum, s8 roomNum) {
+    SceneAndRoom sr;
+    sr.values.sceneIndex = sceneNum;
+    sr.values.roomIndex = roomNum;
+
+    return (VirtualActorList*)&aa->vaZones[sr.raw];
+}
+
+AccessibleActor* ActorAccessibility_AddVirtualActor(VirtualActorList* list, VIRTUAL_ACTOR_TABLE type, Vec3f where) {
+    ActorAccessibilityPolicy* policy = ActorAccessibility_GetPolicyForActor(type);
+    AccessibleActor actor;
+    actor.actor = nullptr;
+    actor.basePitch = 1.0;
+    actor.baseVolume = 1.0;
+    actor.currentPitch = 1.0;
+    actor.currentVolume = 1.0;
+    actor.frameCount = 0;
+    actor.id = (s16)type;
+    actor.instanceID = ActorAccessibility_GetNextID();
+    actor.isDrawn = 1;
+    actor.play = nullptr;
+    actor.pos = where;
+    actor.sceneIndex = 0;
+    actor.managedSoundSlots = 0;
+    actor.aimFramesSinceAimAssist = 0;
+    actor.aimFrequency = 10;
+    actor.policy = *policy;
+
+    VAList_t* l = (VAList_t*)list;
+    l->push_back(actor);
+    return &(*l)[l->size() - 1];
+}
+
+void ActorAccessibility_InterpretCurrentScene(PlayState* play) {
+    if (aa->sceneList.contains(play->sceneNum))
+        return; // Scene interpretation already complete for this scene
+    aa->sceneList.insert(play->sceneNum);
+    VirtualActorList* list = ActorAccessibility_GetVirtualActorList(play->sceneNum, -1); // Scene-global VAs
+    if (list == NULL)
+        return;
+    for (int i = 0; i < play->colCtx.colHeader->numPolygons; i++) {
+        CollisionPoly* poly = &play->colCtx.colHeader->polyList[i];
+        // checks if climable
+        if ((func_80041DB8(&play->colCtx, poly, BGCHECK_SCENE) == 8 ||
+             func_80041DB8(&play->colCtx, poly, BGCHECK_SCENE) == 3)) {
+            ActorAccessibility_PolyToVirtualActor(play, poly, VA_CLIMB, list);
+        }
+        if (SurfaceType_GetSceneExitIndex(&play->colCtx, poly, BGCHECK_SCENE) != 0) {
+            ActorAccessibility_PolyToVirtualActor(play, poly, VA_AREA_CHANGE, list);
+        }
+    }
+}
+
+// Convert poly to VA.
+void ActorAccessibility_PolyToVirtualActor(PlayState* play, CollisionPoly* poly, VIRTUAL_ACTOR_TABLE va,
+                                           VirtualActorList* destination) {
+    Vec3f polyVerts[3];
+    CollisionPoly_GetVertices(poly, play->colCtx.colHeader->vtxList, polyVerts);
+    Vec3f where;
+    where.y = std::min(polyVerts[0].y, std::min(polyVerts[1].y, polyVerts[2].y));
+    f32 minX = std::min(polyVerts[0].x, std::min(polyVerts[1].x, polyVerts[2].x));
+    f32 maxX = std::max(polyVerts[0].x, std::max(polyVerts[1].x, polyVerts[2].x));
+    f32 minZ = std::min(polyVerts[0].z, std::min(polyVerts[1].z, polyVerts[2].z));
+    f32 maxZ = std::max(polyVerts[0].z, std::max(polyVerts[1].z, polyVerts[2].z));
+    where.x = maxX - ((maxX - minX) / 2);
+    where.z = maxZ - ((maxZ - minZ) / 2);
+    AccessibleActor* actor = ActorAccessibility_AddVirtualActor(destination, va, where);
+    if (actor == NULL)
+        return;
+    if (va == VA_AREA_CHANGE) {
+        if (play->sceneNum != SCENE_GROTTOS && play->sceneNum != SCENE_FAIRYS_FOUNTAIN) {
+            u32 sceneIndex = SurfaceType_GetSceneExitIndex(&play->colCtx, poly, BGCHECK_SCENE);
+            s16 nextEntranceIndex = play->setupExitList[sceneIndex - 1];
+            actor->sceneIndex = gEntranceTable[nextEntranceIndex].scene;
+        }
+    }
+}
+
+void ActorAccessibility_AnnounceRoomNumber(PlayState* play) {
+    std::stringstream ss;
+    ss << "Room" << (int)play->roomCtx.curRoom.num;
+    if (Flags_GetClear(play, play->roomCtx.curRoom.num))
+        ss << " completed" << std::endl;
+    SpeechSynthesizer::Instance->Speak(ss.str().c_str(), GetLanguageCode());
+}
+
+AimAssistProps ActorAccessibility_ProvideAimAssistForActor(AccessibleActor* actor) {
+    Player* player = GET_PLAYER(actor->play);
+    s32 angle = player->actor.focus.rot.x;
+    angle = angle / -14000.0 * 16384;
+    f32 cos_angle = Math_CosS(angle);
+    f32 slope = cos_angle == 0.0f ? 0.0f : Math_SinS(angle) / cos_angle;
+    s32 yIntercept = slope * actor->xzDistToPlayer + player->actor.focus.pos.y;
+    s32 yHeight = actor->pos.y + 25;
+    AimAssistProps aimAssistProps;
+    if (yIntercept > yHeight + 25) {
+        aimAssistProps.pitch = 1.5;
+    } else if (yIntercept < yHeight - 25) {
+        aimAssistProps.pitch = 0.5;
+    } else {
+        aimAssistProps.pitch = 1.0;
+    }
+    s32 yDiff = fabs(yIntercept - yHeight);
+    if (yIntercept - yHeight > 0) {
+        s32 correction = 100.0f - 100.0f / std::max(slope, 1.0f);
+        yDiff = std::max(yDiff - correction, 0);
+    }
+    if (yDiff > 300) {
+        actor->aimFrequency = 30;
+    } else {
+        actor->aimFrequency = 1 + (uint8_t)(yDiff / 5);
+    }
+    s16 yawdiff =
+        player->yaw - Math_Atan2S(actor->pos.z - player->actor.world.pos.z, actor->pos.x - player->actor.world.pos.x);
+    if (yawdiff > -0x1000 && yawdiff < 0x1000) {
+        aimAssistProps.volume = 1.0 - (yawdiff * yawdiff) / (float)0x2000000;
+    } else if (yawdiff > -0x2000 && yawdiff < 0x2000) {
+        aimAssistProps.volume = 0.4;
+    } else {
+        aimAssistProps.volume = 0.2;
+    }
+    aimAssistProps.pan = std::min(std::max(yawdiff / (float)0x1000, -1.0f), 1.0f);
+    return aimAssistProps;
+}
+
+// External audio engine stuff.
+bool ActorAccessibility_InitAudio() {
+    try {
+        aa->audioEngine = new AccessibleAudioEngine();
+    } catch (...) {
+        aa->audioEngine = NULL;
+        return false;
+    }
+    return true;
+}
+
+void ActorAccessibility_ShutdownAudio() {
+    if (aa->isOn) {
+        delete aa->audioEngine;
+        if (aa->terrainCues) {
+            DeleteTerrainCueState(aa->terrainCues);
+        }
+        aa->isOn = false;
+    }
+}
+
+void ActorAccessibility_MixAccessibleAudioWithGameAudio(int16_t* ogBuffer, uint32_t nFrames) {
+    if (aa->isOn) {
+        aa->audioEngine->mix(ogBuffer, nFrames);
+    }
+}
+
+// Map one of the game's sfx to a path which as understood by the external audio engine. The returned token is a
+// short hex string that can be passed directly to the audio engine.
+const char* ActorAccessibility_MapSfxToExternalAudio(s16 sfxId) {
+    SfxRecord* record;
+    auto it = aa->sfxMap.find(sfxId);
+    if (it == aa->sfxMap.end()) {
+        SfxRecord tempRecord;
+        std::string fullPath = SfxExtractor::getExternalFileName(sfxId);
+        auto res = Ship::Context::GetInstance()->GetResourceManager()->GetArchiveManager()->LoadFile(fullPath);
+
+        if (res == nullptr)
+            return NULL; // Resource doesn't exist, user's gotta run the extractor.
+        tempRecord.resource = res;
+        std::stringstream ss;
+        ss << std::setw(4) << std::setfill('0') << std::hex << sfxId;
+        tempRecord.path = ss.str();
+        auto pair = aa->sfxMap.insert({ sfxId, tempRecord });
+        record = &pair.first->second;
+        ma_resource_manager_register_decoded_data(&aa->audioEngine->resourceManager, record->path.c_str(),
+                                                  record->resource->Buffer->data(),
+                                                  record->resource->Buffer->size() / 2, ma_format_s16, 1, 44100);
+    } else {
+        record = &it->second;
+    }
+
+    return record->path.c_str();
+}
+
+// Call once per frame to tell the audio engine to start working on the latest batch of queued instructions.
+void ActorAccessibility_PrepareNextAudioFrame() {
+    aa->audioEngine->prepare();
+}
+
+void ActorAccessibility_HandleSoundExtractionMode(PlayState* play) {
+    aa->sfxExtractor.frameCallback();
+}
+
+void ActorAccessibility_DoSoundExtractionStep() {
+    aa->sfxExtractor.captureCallback();
+}

soh/soh/Enhancements/accessible-actors/ActorAccessibility.h

@@ -0,0 +1,180 @@
+#pragma once
+#include <z64.h>
+
+struct AccessibleActor;
+// A callback that is run regularely as the game progresses in order to provide accessibility services for an actor.
+
+typedef void (*ActorAccessibilityCallback)(AccessibleActor*);
+
+struct VirtualActorList;
+
+#define AIM_ALL 0x0F
+#define AIM_BOW 0x01
+#define AIM_SLING 0x02
+#define AIM_SHOOT 0x03
+#define AIM_HOOK 0x04
+#define AIM_BOOM 0x08
+#define AIM_CUP 0x10
+
+struct ActorAccessibilityPolicy {
+    const char* englishName;
+    ActorAccessibilityCallback callback; // If set, it will be called once every n frames.
+                                         // If null, then sfx will be played once every n frames.
+    s16 sound;                           // The ID of a sound to play. Ignored if the callback is set.
+    bool runsAlways;                     // If set, then the distance policy is ignored.
+
+    int n;        // How often to run the callback in frames.
+    f32 distance; // Maximum xz distance from player before the actor should be considered out of range.
+    f32 ydist;    // Maximum y distance from player before the actor should be considered out of range.
+    f32 pitch;
+    f32 volume;
+    f32 pitchModifier;
+    // Aim assist settings.
+    struct {
+        u8 isProvider; // determines whether or not this actor supports aim assist.
+        s16 sfx;       // The sound to play when this actor provides aim assist. Uses sound slot 9.
+        f32 tolerance; // How close to center of actor does Link have to aim to consider it lined up.
+    } aimAssist;
+};
+
+// Accessible actor object. This can be a "real" actor (one that corresponds to an actual actor in the game) or a
+// "virtual" actor (which does not actually exist in the game, but is used to create extra sounds for the player).
+// One potential use of virtual actors is to place sounds at static platforms or other things that aren't represented by
+// actors.
+
+struct AccessibleActor {
+    uint64_t instanceID;
+
+    Actor* actor; // null for virtual actors
+    s16 id;       // For real actors, copy actor ID. For virtual actors we have our own table of values which
+                  // are out of range for real actors.
+    f32 yDistToPlayer;
+    f32 xzDistToPlayer;
+    f32 xyzDistToPlayer;
+    Vec3f pos;
+    Vec3f projectedPos;
+    PlayState* play;
+    u8 isDrawn; // Do we just never play accessibility sounds for actors that aren't drawn?
+
+    u16 frameCount; // Incremented every time the callback is called. The callback is free to modify this. Can be used
+                    // to implement playback of sounds at regular intervals.
+    f32 baseVolume;
+    f32 currentVolume;
+    f32 basePitch;
+
+    f32 currentPitch;
+    s16 sceneIndex; // If this actor represents a scene transition, then this will contain the destination scene index.
+                    // Zero otherwise.
+    u8 managedSoundSlots; // These have their attenuation and panning parameters updated every frame automatically.
+    u8 aimFramesSinceAimAssist; // Used for rate-based vertical aim assist.
+    u8 aimFrequency; // How often the sound will be played. Lower frequencies indicate vertical aim is getting closer.
+
+    ActorAccessibilityPolicy policy; // A copy, so it can be customized on a per-actor basis if needed.
+};
+
+struct AimAssistProps {
+    f32 pitch;
+    f32 volume;
+    f32 pan;
+};
+
+struct TerrainCueState;
+void DeleteTerrainCueState(TerrainCueState*);
+TerrainCueState* InitTerrainCueState(AccessibleActor*);
+void RunTerrainCueState(TerrainCueState*, PlayState*);
+
+// Initialize accessibility.
+void ActorAccessibility_Init();
+void ActorAccessibility_InitActors();
+void ActorAccessibility_Shutdown();
+void ActorAccessibility_InitPolicy(ActorAccessibilityPolicy* policy, const char* englishName);
+void ActorAccessibility_InitPolicy(ActorAccessibilityPolicy* policy, const char* englishName,
+                                   ActorAccessibilityCallback callback);
+void ActorAccessibility_InitPolicy(ActorAccessibilityPolicy* policy, const char* englishName, s16 sfx);
+
+uint64_t ActorAccessibility_GetNextID();
+void ActorAccessibility_TrackNewActor(Actor* actor);
+void ActorAccessibility_RemoveTrackedActor(Actor* actor);
+void ActorAccessibility_AddSupportedActor(s16 type, ActorAccessibilityPolicy policy);
+void ActorAccessibility_AddTerrainCues(AccessibleActor* actor);
+
+void ActorAccessibility_RunAccessibilityForActor(PlayState* play, AccessibleActor* actor);
+void ActorAccessibility_RunAccessibilityForAllActors(PlayState* play);
+/*
+ *Play sounds (usually from the game) using the external sound engine. This is probably not the function you want to
+ *call most of the time (see below). handle: pointer to an arbitrary object. This object can be anything as it's only
+ *used as a classifier, but it's recommended that you use an AccessibleActor* as your handle whenever possible. Using
+ *AccessibleActor* as the handle gives you automatic cleanup when the actor is killed. slot: Allows multiple sounds to
+ *be assigned to a single handle. The maximum number of slots per actor is 10 by default (but can be controlled by
+ *modifying AAE_SLOTS_PER_HANDLE). sfxId: one of the game's sfx IDs. Note that this plays prerendered sounds which you
+ *must have previously prepared. looping: whether to play the sound just once or on a continuous loop.
+ */
+void ActorAccessibility_PlaySound(void* actor, int slot, s16 sfxId);
+// Stop a sound. Todo: consider making this a short fade instead of just cutting it off.
+void ActorAccessibility_StopSound(void* handle, int slot);
+void ActorAccessibility_StopAllSounds(void* handle);
+
+void ActorAccessibility_SetSoundPitch(void* handle, int slot, float pitch);
+// When we don't have access to something super fancy (such as HRTF), blind-accessible games generally use a change in
+// pitch to tell the player that an object is behind the player.
+void ActorAccessibility_SetPitchBehindModifier(void* handle, int slot, float mod);
+
+void ActorAccessibility_SetListenerPos(Vec3f* pos, Vec3f* rot);
+void ActorAccessibility_SetSoundPos(void* handle, int slot, Vec3f* pos, f32 distToPlayer, f32 maxDistance);
+
+void ActorAccessibility_SetSoundVolume(void* handle, int slot, float volume);
+void ActorAccessibility_SetSoundPan(void* handle, int slot, float pan);
+void ActorAccessibility_SetSoundFilter(void* handle, int slot, float cutoff);
+void ActorAccessibility_SeekSound(void* handle, int slot, size_t offset);
+
+/*
+ * Play a sound on behalf of an AccessibleActor.
+ * This version includes automatic sound management: pitch, panning and attenuation parameters will be updated
+ * automatically based on the actor's position.
+ *
+ */
+void ActorAccessibility_PlaySoundForActor(AccessibleActor* actor, int slot, s16 sfxId);
+
+void ActorAccessibility_StopSoundForActor(AccessibleActor* actor, int slot);
+void ActorAccessibility_StopAllSoundsForActor(AccessibleActor* actor);
+f32 ActorAccessibility_ComputeCurrentVolume(f32 maxDistance, f32 xzDistToPlayer);
+// Computes a relative angle based on Link's (or some other actor's) current angle.
+Vec3s ActorAccessibility_ComputeRelativeAngle(Vec3s* origin, Vec3s* offset);
+// Stuff related to lists of virtual actors.
+typedef enum {
+    // Similar to the game's actual actor table
+    VA_INITIAL = 1000,
+    VA_CRAWLSPACE,
+    VA_TERRAIN_CUE,
+    VA_WALL_CUE,
+    VA_CLIMB,
+    VA_DOOR,
+    VA_AREA_CHANGE,
+    VA_MARKER,
+    VA_MAX,
+} VIRTUAL_ACTOR_TABLE;
+
+// Get the list of virtual actors for a given scene and room index.
+VirtualActorList* ActorAccessibility_GetVirtualActorList(s16 sceneNum, s8 roomNum);
+AccessibleActor* ActorAccessibility_AddVirtualActor(VirtualActorList* list, VIRTUAL_ACTOR_TABLE type, Vec3f where);
+// Parses the loaded seen and converts select polygons (like ladders, spikes and scene exits) into virtual actors.
+void ActorAccessibility_InterpretCurrentScene(PlayState* play);
+// Convert a collision polygon into a virtual actor.
+void ActorAccessibility_PolyToVirtualActor(PlayState* play, CollisionPoly* poly, VIRTUAL_ACTOR_TABLE va,
+                                           VirtualActorList* destination);
+// Report which room of a dungeon the player is in.
+void ActorAccessibility_AnnounceRoomNumber(PlayState* play);
+// Aim cue support.
+AimAssistProps ActorAccessibility_ProvideAimAssistForActor(AccessibleActor* actor);
+// External audio engine stuff.
+//  Initialize the accessible audio engine.
+bool ActorAccessibility_InitAudio();
+void ActorAccessibility_ShutdownAudio();
+// Combine the games' audio with the output from AccessibleAudioEngine. To be called exclusively from the audio thread.
+void ActorAccessibility_MixAccessibleAudioWithGameAudio(int16_t* ogBuffer, uint32_t nFrames);
+void ActorAccessibility_HandleSoundExtractionMode(PlayState* play);
+// This is called by the audio thread when it's ready to try to pull sfx from the game.
+void ActorAccessibility_DoSoundExtractionStep();
+
+void ActorAccessibility_GeneralHelper(PlayState* play);
+void ActorAccessibility_AudioGlossary(PlayState* play);

soh/soh/Enhancements/accessible-actors/SfxExtractor.cpp

@@ -0,0 +1,218 @@
+#include "SfxExtractor.h"
+#include "soh/Enhancements/audio/miniaudio.h"
+#include "soh/Enhancements/speechsynthesizer/SpeechSynthesizer.h"
+#include "soh/Enhancements/tts/tts.h"
+#include "soh/OTRGlobals.h"
+#include "SfxTable.h"
+#include <sstream>
+extern "C" {
+#include "z64.h"
+#include "functions.h"
+#include "variables.h"
+void AudioMgr_CreateNextAudioBuffer(s16* samples, u32 num_samples);
+extern bool freezeGame;
+}
+
+bool SfxExtractor::isAllSilence(int16_t* buffer, size_t count) {
+    for (size_t i = 0; i < count; i++) {
+        if (!isSilentSample(buffer[i])) // Tolerance for low-amplitude dither noise.
+            return false;
+    }
+    return true;
+}
+bool SfxExtractor::isSilentSample(int16_t sample) {
+    return abs(sample) <= SFX_EXTRACTION_SILENCE_THRESHOLD;
+}
+
+// Find the beginning of a captured signal.
+size_t SfxExtractor::adjustedStartOfInput() {
+    size_t startOfInput = 0;
+    while (startOfInput + 2 < SFX_EXTRACTION_BUFFER_SIZE * 2 && isSilentSample(tempBuffer[startOfInput]) &&
+           isSilentSample(tempBuffer[startOfInput + 1])) {
+        startOfInput += 2;
+    }
+    return startOfInput;
+}
+
+size_t SfxExtractor::adjustedEndOfInput(size_t endOfInput) {
+    while (endOfInput > 0 && (!isSilentSample(tempBuffer[endOfInput]) || isSilentSample(tempBuffer[endOfInput - 1]))) {
+        endOfInput -= 2;
+    }
+    return endOfInput;
+}
+
+bool SfxExtractor::renderOutput(size_t endOfInput) {
+    size_t startOfInput = adjustedStartOfInput();
+    endOfInput = adjustedEndOfInput(endOfInput);
+    if (endOfInput <= startOfInput) {
+        return true;
+    }
+
+    ma_channel_converter_config config =
+        ma_channel_converter_config_init(ma_format_s16, 2, NULL, 1, NULL, ma_channel_mix_mode_default);
+    ma_channel_converter converter;
+    if (ma_channel_converter_init(&config, NULL, &converter) != MA_SUCCESS) {
+        return false;
+    }
+    std::vector<uint8_t> fileData;
+    std::string fileName = getExternalFileName(currentSfx);
+    int16_t chunk[64];
+    int16_t* mark = tempBuffer + startOfInput;
+    while (mark < tempBuffer + endOfInput) {
+        size_t chunkSize = std::min<size_t>(64, ((tempBuffer + endOfInput) - mark) / 2);
+        ma_result converter_result = ma_channel_converter_process_pcm_frames(&converter, chunk, mark, chunkSize);
+        if (converter_result != MA_SUCCESS) {
+            return false;
+        }
+        fileData.insert(fileData.end(), (uint8_t*)chunk, (uint8_t*)(chunk + chunkSize));
+        mark += chunkSize * 2;
+    }
+    return archive->WriteFile(fileName.c_str(), fileData);
+}
+
+void SfxExtractor::setup() {
+    try {
+        SpeechSynthesizer::Instance->Speak(
+            "Sfx extraction speedrun initiated. Please wait. This will take a few minutes.", "en-US");
+        // Kill the audio thread so we can take control.
+        captureThreadState = CT_WAITING;
+        OTRAudio_InstallSfxCaptureThread();
+        // Make sure we're starting from a clean slate.
+        std::string sohAccessibilityPath = Ship::Context::GetPathRelativeToAppBundle("accessibility.o2r");
+        if (std::filesystem::exists(sohAccessibilityPath)) {
+            currentStep = STEP_ERROR_FILE_EXISTS;
+            return;
+        }
+
+        sfxToRip = 0;
+        currentSfx = -1;
+        currentStep = STEP_MAIN;
+        archive = std::make_shared<Ship::O2rArchive>(sohAccessibilityPath);
+        archive->Open();
+    } catch (...) { currentStep = STEP_ERROR; }
+}
+
+void SfxExtractor::ripNextSfx() {
+    // This entire method is expected to be atomic; Don't try to narrow the scope of this lock please!
+    // Todo: remove the thread altogether as we don't actually need or want parallelism here.
+    auto lock = OTRAudio_Lock();
+    if (captureThreadState == CT_READY || captureThreadState == CT_PRIMING)
+        return; // Keep going.
+    // Was the last sfx a loop? If so then we need to stop it, and then we need to run audio out to nowhere for as long
+    // as it takes to get back to a blank slate.
+    if (currentSfx != -1) {
+        Audio_StopSfxByPos(&gSfxDefaultPos);
+        captureThreadState = CT_PRIMING;
+        currentSfx = -1;
+
+        return;
+    }
+    if (sfxToRip == sfxCount) {
+        currentStep = STEP_FINISHED; // Caught 'em all!
+        return;
+    }
+
+    currentSfx = sfxTable[sfxToRip++];
+    Audio_PlaySoundGeneral(currentSfx, &gSfxDefaultPos, 4, &gSfxDefaultFreqAndVolScale, &gSfxDefaultFreqAndVolScale,
+                           &gSfxDefaultReverb);
+    captureThreadState = CT_READY;
+    maybeGiveProgressReport();
+}
+void SfxExtractor::finished() {
+    OTRAudio_UninstallSfxCaptureThread(); // Returns to normal audio opperation.
+    archive->Close();
+    archive = nullptr;
+    freezeGame = false;
+
+    Audio_QueueSeqCmd(NA_BGM_TITLE);
+
+    if (currentStep >= STEP_ERROR) {
+        Audio_PlaySoundGeneral(NA_SE_SY_ERROR, &gSfxDefaultPos, 4, &gSfxDefaultFreqAndVolScale,
+                               &gSfxDefaultFreqAndVolScale, &gSfxDefaultReverb);
+        Audio_PlaySoundGeneral(NA_SE_EN_GANON_LAUGH, &gSfxDefaultPos, 4, &gSfxDefaultFreqAndVolScale,
+                               &gSfxDefaultFreqAndVolScale, &gSfxDefaultReverb);
+        std::stringstream ss;
+        ss << "Sorry, we tried to extract the sound effects, but Ganondorf overruled us with an iron fist."
+           << std::endl;
+        if (currentStep == STEP_ERROR_FILE_EXISTS)
+            ss << "In all seriousness, please delete accessibility.o2r and try again.";
+        SpeechSynthesizer::Instance->Speak(ss.str().c_str(), "en-US");
+    } else
+        Audio_PlayFanfare(NA_BGM_ITEM_GET);
+}
+void SfxExtractor::maybeGiveProgressReport() {
+    for (int i = 0; i < 9; i++) {
+        if (sfxToRip == sfxCount * (i + 1) / 10) {
+            std::stringstream ss;
+            ss << (i + 1) * 10 << " percent complete.";
+            SpeechSynthesizer::Instance->Speak(ss.str().c_str(), "en-US");
+        }
+    }
+}
+
+SfxExtractor::SfxExtractor() {
+    currentStep = STEP_SETUP;
+}
+
+void SfxExtractor::frameCallback() {
+    switch (currentStep) {
+        case STEP_SETUP:
+            setup();
+            break;
+        case STEP_MAIN:
+            ripNextSfx();
+            break;
+        default: // Handles finished as well as a number of error conditions.
+            finished();
+    }
+}
+
+void SfxExtractor::prime() {
+    int frameLimit = 0; // A couple of sounds don't come to a full stop until another sound is loaded, but should be
+                        // effectively silent after a couple of seconds.
+    do {
+        AudioMgr_CreateNextAudioBuffer(tempBuffer + 0, SFX_EXTRACTION_ONE_FRAME);
+    } while (frameLimit++ < 200 && !isAllSilence(tempBuffer + 0, SFX_EXTRACTION_ONE_FRAME * 2));
+    captureThreadState = CT_FINISHED;
+}
+
+void SfxExtractor::captureCallback() {
+    if (captureThreadState == CT_PRIMING)
+        prime();
+    if (captureThreadState != CT_READY)
+        return; // No work to do at the moment.
+    memset(tempBuffer, 0, sizeof(tempBuffer));
+    int16_t* mark = tempBuffer + 0;
+    size_t samplesLeft = SFX_EXTRACTION_BUFFER_SIZE;
+    bool outputStarted = false;
+    int waitTime = 0;
+    while (samplesLeft > 0) {
+        AudioMgr_CreateNextAudioBuffer(mark, SFX_EXTRACTION_ONE_FRAME);
+
+        if (isAllSilence(mark, SFX_EXTRACTION_ONE_FRAME * 2)) {
+            if (outputStarted) {
+                break;
+            } else if (waitTime++ < 300) {
+                continue; // Output is silent, allow more time for audio to begin.
+            }
+            captureThreadState = CT_FINISHED; // Sound is unavailable, so skip over it and move on.
+            return;
+        }
+
+        outputStarted = true;
+        size_t samples = std::min<size_t>(SFX_EXTRACTION_ONE_FRAME, samplesLeft);
+        mark += samples * 2;
+        samplesLeft -= samples;
+    }
+    if (renderOutput(mark - tempBuffer)) {
+        captureThreadState = CT_FINISHED;
+    } else {
+        SPDLOG_ERROR("failed to write file to archive, trying again");
+    }
+}
+
+std::string SfxExtractor::getExternalFileName(int16_t sfxId) {
+    std::stringstream ss;
+    ss << "accessibility/audio/" << std::hex << std::setw(4) << std::setfill('0') << sfxId << ".wav";
+    return ss.str();
+}

soh/soh/Enhancements/accessible-actors/SfxExtractor.h

@@ -0,0 +1,51 @@
+#pragma once
+#include "libultraship/libultraship.h"
+
+#define SFX_EXTRACTION_BUFFER_SIZE 32000 * 15
+#define SFX_EXTRACTION_ONE_FRAME 560
+#define SFX_EXTRACTION_SILENCE_THRESHOLD 6 // Corresponds to an amplitude of -75dB.
+
+enum CaptureThreadStates {
+    CT_WAITING, // for a sound to start ripping.
+    CT_PRIMING,
+    CT_READY,    // to start ripping a sound.
+    CT_FINISHED, // ripping the current sound.
+    CT_SHUTDOWN,
+};
+
+enum SfxExtractionSteps {
+    STEP_SETUP = 0,
+    STEP_MAIN,
+    STEP_FINISHED,
+    STEP_ERROR,
+    STEP_ERROR_FILE_EXISTS,
+};
+
+class SfxExtractor {
+    std::shared_ptr<Ship::Archive> archive;
+    SfxExtractionSteps currentStep;
+    CaptureThreadStates captureThreadState;
+    int sfxToRip;
+    s16 currentSfx;
+    // Stores raw audio data for the sfx currently being ripped.
+    int16_t tempBuffer[(SFX_EXTRACTION_BUFFER_SIZE + SFX_EXTRACTION_ONE_FRAME * 3) * 2];
+    // Check if a buffer contains meaningful audio output.
+    bool isAllSilence(int16_t* buffer, size_t count);
+    bool isSilentSample(int16_t sample);
+    size_t adjustedStartOfInput();
+    size_t adjustedEndOfInput(size_t endOfInput);
+    bool renderOutput(size_t endOfInput);
+    void setup();
+    void ripNextSfx();
+    void finished(); // Also handles failure.
+    void maybeGiveProgressReport();
+
+  public:
+    SfxExtractor();
+
+    void frameCallback();
+    void prime();
+    // The below is called by the (hijacked) audio thread.
+    void captureCallback();
+    static std::string getExternalFileName(int16_t sfxId);
+};

soh/soh/Enhancements/audio/miniaudio.h

@@ -0,0 +1,9 @@
+#pragma once
+#define MA_NO_FLAC
+#define MA_NO_MP3
+#define MA_NO_THREADING
+#define MA_NO_DEVICE_IO
+#define MA_NO_GENERATION
+#define MA_NO_STDIO
+#define MA_ENABLE_ONLY_SPECIFIC_BACKENDS
+#include <miniaudio.h>

soh/soh/Enhancements/game-interactor/GameInteractor_HookTable.h

@@ -70,6 +70,7 @@ DEFINE_HOOK(OnUpdateFileNameSelection, (int16_t charCode));
 DEFINE_HOOK(OnFileChooseMain, (void* gameState));
 
 DEFINE_HOOK(OnSetGameLanguage, ());
+DEFINE_HOOK(OnGameStillFrozen, ());
 DEFINE_HOOK(OnFileDropped, (std::string filePath));
 DEFINE_HOOK(OnAssetAltChange, ());
 DEFINE_HOOK(OnKaleidoUpdate, ());

soh/soh/Enhancements/game-interactor/GameInteractor_Hooks.cpp

@@ -312,6 +312,10 @@ void GameInteractor_ExecuteOnSetGameLanguage() {
     GameInteractor::Instance->ExecuteHooks<GameInteractor::OnSetGameLanguage>();
 }
 
+void GameInteractor_ExecuteOnGameStillFrozen() {
+    GameInteractor::Instance->ExecuteHooks<GameInteractor::OnGameStillFrozen>();
+}
+
 // MARK: - System
 
 void GameInteractor_RegisterOnAssetAltChange(void (*fn)(void)) {

soh/soh/Enhancements/game-interactor/GameInteractor_Hooks.h

@@ -79,6 +79,8 @@ void GameInteractor_ExecuteOnFileChooseMain(void* gameState);
 // MARK: - Game
 void GameInteractor_ExecuteOnSetGameLanguage();
 
+void GameInteractor_ExecuteOnGameStillFrozen();
+
 // MARK: - System
 void GameInteractor_RegisterOnAssetAltChange(void (*fn)(void));
 

soh/soh/OTRGlobals.cpp

@@ -127,6 +127,9 @@ Sail* Sail::Instance;
 
 #include "soh/config/ConfigUpdaters.h"
 #include "soh/ShipInit.hpp"
+#if !defined(__SWITCH__) && !defined(__WIIU__)
+#include "Enhancements/accessible-actors/ActorAccessibility.h"
+#endif
 
 extern "C" {
 #include "src/overlays/actors/ovl_En_Dns/z_en_dns.h"
@@ -295,6 +298,12 @@ void OTRGlobals::Initialize() {
             }
         }
     }
+
+    std::string sohAccessibilityPath = Ship::Context::GetPathRelativeToAppBundle("accessibility.o2r");
+    if (std::filesystem::exists(sohAccessibilityPath)) {
+        OTRFiles.push_back(sohAccessibilityPath);
+    }
+
     std::sort(patchOTRs.begin(), patchOTRs.end(), [](const std::string& a, const std::string& b) {
         return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end(),
                                             [](char c1, char c2) { return std::tolower(c1) < std::tolower(c2); });
@@ -588,6 +597,11 @@ void OTRAudio_Thread() {
         for (int i = 0; i < AUDIO_FRAMES_PER_UPDATE; i++) {
             AudioMgr_CreateNextAudioBuffer(audio_buffer + i * (num_audio_samples * NUM_AUDIO_CHANNELS),
                                            num_audio_samples);
+#if !defined(__SWITCH__) && !defined(__WIIU__)
+            // Give accessibility a chance to merge its own audio in.
+            ActorAccessibility_MixAccessibleAudioWithGameAudio(
+                audio_buffer + i * (num_audio_samples * NUM_AUDIO_CHANNELS), num_audio_samples);
+#endif
         }
 
         AudioPlayer_Play((u8*)audio_buffer,
@@ -1277,6 +1291,9 @@ extern "C" void InitOTR() {
 #endif
 
     OTRMessage_Init();
+#if !defined(__SWITCH__) && !defined(__WIIU__)
+    ActorAccessibility_Init();
+#endif
     OTRAudio_Init();
     OTRExtScanner();
     VanillaItemTable_Init();
@@ -1328,6 +1345,9 @@ extern "C" void DeinitOTR() {
     }
     SDLNet_Quit();
 #endif
+#if !defined(__SWITCH__) && !defined(__WIIU__)
+    ActorAccessibility_Shutdown();
+#endif
 
     // Destroying gui here because we have shared ptrs to LUS objects which output to SPDLOG which is destroyed before
     // these shared ptrs.
@@ -2606,6 +2626,40 @@ extern "C" void Gfx_RegisterBlendedTexture(const char* name, u8* mask, u8* repla
     }
 }
 
+void OTRAudio_SfxCaptureThread() {
+    while (audio.running) {
+        // This entire body is expected to be atomic; Don't try to narrow the scope of this lock please!
+        // Todo: remove the thread altogether as we don't actually need or want parallelism here.
+        std::unique_lock<std::mutex> Lock(audio.mutex);
+        while (!audio.processing && audio.running) {
+            audio.cv_to_thread.wait(Lock);
+        }
+
+        if (!audio.running) {
+            break;
+        }
+#if !defined(__SWITCH__) && !defined(__WIIU__)
+        ActorAccessibility_DoSoundExtractionStep();
+#endif
+        audio.processing = false;
+        audio.cv_from_thread.notify_one();
+    }
+}
+
+extern "C" void OTRAudio_InstallSfxCaptureThread() {
+    OTRAudio_Exit();
+    audio.running = true;
+    audio.thread = std::thread(OTRAudio_SfxCaptureThread);
+}
+extern "C" void OTRAudio_UninstallSfxCaptureThread() {
+    OTRAudio_Exit();
+    audio.running = true;
+    audio.thread = std::thread(OTRAudio_Thread);
+}
+std::unique_lock<std::mutex> OTRAudio_Lock() {
+    return std::unique_lock<std::mutex>(audio.mutex);
+}
+
 extern "C" void Gfx_UnregisterBlendedTexture(const char* name) {
     if (auto intP = dynamic_pointer_cast<Fast::Fast3dWindow>(Ship::Context::GetInstance()->GetWindow())
                         ->GetInterpreterWeak()

soh/soh/OTRGlobals.h

@@ -178,8 +178,12 @@ void Messagebox_ShowErrorBox(char* title, char* body);
 extern "C" {
 #endif
 uint64_t GetUnixTimestamp();
+void OTRAudio_InstallSfxCaptureThread();
+void OTRAudio_UninstallSfxCaptureThread();
 #ifdef __cplusplus
 };
+std::unique_lock<std::mutex> OTRAudio_Lock();
+
 #endif
 
 #endif

soh/soh/SohGui/SohMenuSettings.cpp

@@ -233,6 +233,12 @@ void SohMenu::AddMenuSettings() {
         .CVar(CVAR_SETTING("A11yDisableIdleCam"))
         .RaceDisable(false)
         .Options(CheckboxOptions().Tooltip("Disables the automatic re-centering of the camera when idle."));
+
+    AddWidget(path, "Accessible Audio Cues", WIDGET_CVAR_CHECKBOX)
+        .CVar(CVAR_SETTING("A11yAudioInteraction"))
+        .RaceDisable(false)
+        .Options(CheckboxOptions().Tooltip("Enables accessibility audio cues"));
+
     AddWidget(path, "EXPERIMENTAL", WIDGET_SEPARATOR_TEXT).Options(TextOptions().Color(Colors::Orange));
     AddWidget(path, "ImGui Menu Scaling", WIDGET_CVAR_COMBOBOX)
         .CVar(CVAR_SETTING("ImGuiScale"))

soh/src/code/code_800F9280.c

@@ -3,6 +3,7 @@
 #include "soh/mixer.h"
 
 #include "soh/Enhancements/audio/AudioEditor.h"
+extern bool freezeGame;
 
 typedef struct {
     u8 unk_0;
@@ -372,6 +373,9 @@ extern f32 D_80130F24;
 extern f32 D_80130F28;
 
 void Audio_QueueSeqCmd(u32 cmd) {
+    if (freezeGame)
+        return; // No music during SFX rip.
+                //
     u8 op = cmd >> 28;
     if (op == 0 || op == 2 || op == 12) {
         u8 seqId = cmd & 0xFF;

soh/src/code/z_actor.c

@@ -82,6 +82,7 @@
 #include "textures/place_title_cards/g_pn_56.h"
 #include "textures/place_title_cards/g_pn_57.h"
 #endif
+bool freezeActors = false;
 
 static CollisionPoly* sCurCeilingPoly;
 static s32 sCurCeilingBgId;
@@ -1264,6 +1265,7 @@ void Actor_Init(Actor* actor, PlayState* play) {
 }
 
 void Actor_Destroy(Actor* actor, PlayState* play) {
+    GameInteractor_ExecuteOnActorDestroy(actor);
     if (actor->destroy != NULL) {
         actor->destroy(actor, play);
         actor->destroy = NULL;
@@ -2561,6 +2563,7 @@ u32 D_80116068[ACTORCAT_MAX] = {
 };
 
 void Actor_UpdateAll(PlayState* play, ActorContext* actorCtx) {
+
     Actor* refActor;
     Actor* actor;
     Player* player;
@@ -2576,6 +2579,11 @@ void Actor_UpdateAll(PlayState* play, ActorContext* actorCtx) {
     sp74 = NULL;
     unkFlag = 0;
 
+    if (freezeActors) {
+        GameInteractor_ExecuteOnPlayerUpdate();
+        return; // for AudioGlossary
+    }
+
     if (play->numSetupActors != 0) {
         actorEntry = &play->setupActorList[0];
         for (i = 0; i < play->numSetupActors; i++) {

soh/src/code/z_play.c

@@ -21,6 +21,7 @@
 #include <time.h>
 #include <assert.h>
 
+bool freezeGame = false; // Used for SFX ripper.
 TransitionUnk sTrnsnUnk;
 s32 gTrnsnUnkState;
 VisMono gPlayVisMono;
@@ -694,6 +695,11 @@ void Play_Update(PlayState* play) {
     s32 isPaused;
     s32 pad1;
 
+    if (freezeGame) {
+        GameInteractor_ExecuteOnGameStillFrozen();
+        return;
+    }
+
     if ((SREG(1) < 0) || (DREG(0) != 0)) {
         SREG(1) = 0;
         ZeldaArena_Display();