Add Android Output using Oboe

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "third-party/gf-complete"]
 	path = third-party/gf-complete
 	url = https://github.com/thestr4ng3r/gf-complete.git
+[submodule "android/app/src/main/cpp/oboe"]
+	path = android/app/src/main/cpp/oboe
+	url = https://github.com/google/oboe

android/app/CMakeLists.txt

@@ -6,10 +6,15 @@ add_library(chiaki-jni SHARED
         src/main/cpp/video-decoder.h
         src/main/cpp/video-decoder.c
         src/main/cpp/audio-decoder.h
-        src/main/cpp/audio-decoder.c)
+        src/main/cpp/audio-decoder.c
+        src/main/cpp/audio-output.h
+        src/main/cpp/audio-output.cpp)
 target_link_libraries(chiaki-jni chiaki-lib)
 
 find_library(ANDROID_LIB_LOG log)
 find_library(ANDROID_LIB_MEDIANDK mediandk)
 find_library(ANDROID_LIB_ANDROID android)
 target_link_libraries(chiaki-jni "${ANDROID_LIB_LOG}" "${ANDROID_LIB_MEDIANDK}" "${ANDROID_LIB_ANDROID}")
+
+add_subdirectory(src/main/cpp/oboe)
+target_link_libraries(chiaki-jni oboe)

android/app/src/main/cpp/audio-decoder.c

@@ -21,7 +21,6 @@
 
 #include <media/NdkMediaCodec.h>
 #include <media/NdkMediaFormat.h>
-#include <android/native_window_jni.h>
 
 #include <string.h>
 
@@ -37,6 +36,11 @@ ChiakiErrorCode android_chiaki_audio_decoder_init(AndroidChiakiAudioDecoder *dec
 	memset(&decoder->audio_header, 0, sizeof(decoder->audio_header));
 	decoder->codec = NULL;
 	decoder->timestamp_cur = 0;
+
+	decoder->cb_user = NULL;
+	decoder->settings_cb = NULL;
+	decoder->frame_cb = NULL;
+
 	return CHIAKI_ERR_SUCCESS;
 }
 
@@ -59,11 +63,17 @@ static void *android_chiaki_audio_decoder_output_thread_func(void *user)
 	while(1)
 	{
 		AMediaCodecBufferInfo info;
-		ssize_t status = AMediaCodec_dequeueOutputBuffer(decoder->codec, &info, -1);
-		if(status >= 0)
+		ssize_t codec_buf_index = AMediaCodec_dequeueOutputBuffer(decoder->codec, &info, -1);
+		if(codec_buf_index >= 0)
 		{
 			CHIAKI_LOGD(decoder->log, "Got decoded audio of size %d", (int)info.size);
-			AMediaCodec_releaseOutputBuffer(decoder->codec, (size_t)status, info.size != 0);
+			if(decoder->settings_cb)
+			{
+				size_t codec_buf_size;
+				uint8_t *codec_buf = AMediaCodec_getOutputBuffer(decoder->codec, (size_t)codec_buf_index, &codec_buf_size);
+				decoder->frame_cb((int16_t *)codec_buf, codec_buf_size / 2, decoder->cb_user);
+			}
+			AMediaCodec_releaseOutputBuffer(decoder->codec, (size_t)codec_buf_index, info.size != 0);
 			if(info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM)
 			{
 				CHIAKI_LOGI(decoder->log, "AMediaCodec for Audio Decoder reported EOS");
@@ -100,6 +110,7 @@ static void android_chiaki_audio_decoder_header(ChiakiAudioHeader *header, void
 	AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, mime);
 	AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_CHANNEL_COUNT, header->channels);
 	AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_SAMPLE_RATE, header->rate);
+	// AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_PCM_ENCODING)
 
 	AMediaCodec_configure(decoder->codec, format, NULL, NULL, 0); // TODO: check result
 	AMediaCodec_start(decoder->codec); // TODO: check result
@@ -142,6 +153,9 @@ static void android_chiaki_audio_decoder_header(ChiakiAudioHeader *header, void
 	uint8_t csd2[8] = { (uint8_t)(pre_roll_ns & 0xff), (uint8_t)((pre_roll_ns >> 0x8) & 0xff), (uint8_t)((pre_roll_ns >> 0x10) & 0xff), (uint8_t)((pre_roll_ns >> 0x18) & 0xff),
 						(uint8_t)((pre_roll_ns >> 0x20) & 0xff), (uint8_t)((pre_roll_ns >> 0x28) & 0xff), (uint8_t)((pre_roll_ns >> 0x30) & 0xff), (uint8_t)(pre_roll_ns >> 0x38)};
 	android_chiaki_audio_decoder_frame(csd2, sizeof(csd2), decoder);
+
+	if(decoder->settings_cb)
+		decoder->settings_cb(header->channels, header->rate, decoder->cb_user);
 }
 
 static void android_chiaki_audio_decoder_frame(uint8_t *buf, size_t buf_size, void *user)

android/app/src/main/cpp/audio-decoder.h

@@ -24,6 +24,9 @@
 #include <chiaki/log.h>
 #include <chiaki/audioreceiver.h>
 
+typedef void (*AndroidChiakiAudioDecoderSettingsCallback)(uint32_t channels, uint32_t rate, void *user);
+typedef void (*AndroidChiakiAudioDecoderFrameCallback)(int16_t *buf, size_t samples_count, void *user);
+
 typedef struct android_chiaki_audio_decoder_t
 {
 	ChiakiLog *log;
@@ -31,10 +34,21 @@ typedef struct android_chiaki_audio_decoder_t
 	struct AMediaCodec *codec;
 	uint64_t timestamp_cur;
 	ChiakiThread output_thread;
+
+	AndroidChiakiAudioDecoderSettingsCallback settings_cb;
+	AndroidChiakiAudioDecoderFrameCallback frame_cb;
+	void *cb_user;
 } AndroidChiakiAudioDecoder;
 
 ChiakiErrorCode android_chiaki_audio_decoder_init(AndroidChiakiAudioDecoder *decoder, ChiakiLog *log);
 void android_chiaki_audio_decoder_fini(AndroidChiakiAudioDecoder *decoder);
 void android_chiaki_audio_decoder_get_sink(AndroidChiakiAudioDecoder *decoder, ChiakiAudioSink *sink);
 
+static inline void android_chiaki_audio_decoder_set_cb(AndroidChiakiAudioDecoder *decoder, AndroidChiakiAudioDecoderSettingsCallback settings_cb, AndroidChiakiAudioDecoderFrameCallback frame_cb, void *user)
+{
+	decoder->settings_cb = settings_cb;
+	decoder->frame_cb = frame_cb;
+	decoder->cb_user = user;
+}
+
 #endif

android/app/src/main/cpp/audio-output.cpp

@@ -0,0 +1,94 @@
+/*
+ * This file is part of Chiaki.
+ *
+ * Chiaki is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Chiaki is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Chiaki.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "audio-output.h"
+
+#include <chiaki/log.h>
+#include <chiaki/thread.h>
+
+#include <oboe/Oboe.h>
+
+#define WRITE_TIMEOUT_MS 8
+
+struct AudioOutput
+{
+	ChiakiLog *log;
+	oboe::ManagedStream stream;
+	ChiakiMutex stream_mutex;
+};
+
+extern "C" void *android_chiaki_audio_output_new(ChiakiLog *log)
+{
+	auto r = new AudioOutput();
+	r->log = log;
+	chiaki_mutex_init(&r->stream_mutex, false);
+	return r;
+}
+
+extern "C" void android_chiaki_audio_output_free(void *audio_output)
+{
+	if(!audio_output)
+		return;
+	auto ao = reinterpret_cast<AudioOutput *>(audio_output);
+	chiaki_mutex_fini(&ao->stream_mutex);
+	delete ao;
+}
+
+extern "C" void android_chiaki_audio_output_settings(uint32_t channels, uint32_t rate, void *audio_output)
+{
+	auto ao = reinterpret_cast<AudioOutput *>(audio_output);
+	ChiakiMutexLock lock(&ao->stream_mutex);
+
+	oboe::AudioStreamBuilder builder;
+	builder.setPerformanceMode(oboe::PerformanceMode::LowLatency)
+		->setSharingMode(oboe::SharingMode::Exclusive)
+		->setFormat(oboe::AudioFormat::I16)
+		->setChannelCount(channels)
+		->setSampleRate(rate);
+
+	auto result = builder.openManagedStream(ao->stream);
+	if(result == oboe::Result::OK)
+		CHIAKI_LOGI(ao->log, "Audio Output opened Oboe stream");
+	else
+		CHIAKI_LOGE(ao->log, "Audio Output failed to open Oboe stream: %s", oboe::convertToText(result));
+
+	result = ao->stream->start();
+	if(result == oboe::Result::OK)
+		CHIAKI_LOGI(ao->log, "Audio Output started Oboe stream");
+	else
+		CHIAKI_LOGE(ao->log, "Audio Output failed to start Oboe stream: %s", oboe::convertToText(result));
+}
+
+extern "C" void android_chiaki_audio_output_frame(int16_t *buf, size_t samples_count, void *audio_output)
+{
+	auto ao = reinterpret_cast<AudioOutput *>(audio_output);
+	ChiakiMutexLock lock(&ao->stream_mutex);
+
+	if(ao->stream->getFormat() != oboe::AudioFormat::I16)
+	{
+		CHIAKI_LOGE(ao->log, "Oboe stream has invalid format for pushing samples");
+		return;
+	}
+
+	int32_t samples_per_frame = ao->stream->getBytesPerFrame() / ao->stream->getBytesPerSample();
+	if(samples_count % samples_per_frame != 0)
+		CHIAKI_LOGW(ao->log, "Received %llu samples, which is not dividable by %llu samples per frame", (unsigned long long)samples_count, (unsigned long long)samples_per_frame);
+
+	size_t frames = samples_count / samples_per_frame;
+	if(frames > 0)
+		ao->stream->write(buf, frames, WRITE_TIMEOUT_MS * 1000 * 1000);
+}

android/app/src/main/cpp/audio-output.h

@@ -0,0 +1,37 @@
+/*
+ * This file is part of Chiaki.
+ *
+ * Chiaki is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Chiaki is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Chiaki.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#ifndef CHIAKI_JNI_AUDIO_OUTPUT_H
+#define CHIAKI_JNI_AUDIO_OUTPUT_H
+
+#include <chiaki/log.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void *android_chiaki_audio_output_new(ChiakiLog *log);
+void android_chiaki_audio_output_free(void *audio_output);
+void android_chiaki_audio_output_settings(uint32_t channels, uint32_t rate, void *audio_output);
+void android_chiaki_audio_output_frame(int16_t *buf, size_t samples_count, void *audio_output);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif //CHIAKI_JNI_AUDIO_OUTPUT_H

android/app/src/main/cpp/chiaki-jni.c

@@ -27,6 +27,7 @@
 
 #include "video-decoder.h"
 #include "audio-decoder.h"
+#include "audio-output.h"
 
 #define LOG_TAG "Chiaki"
 #define JNI_VERSION JNI_VERSION_1_6
@@ -115,6 +116,7 @@ typedef struct android_chiaki_session_t
 
 	AndroidChiakiVideoDecoder video_decoder;
 	AndroidChiakiAudioDecoder audio_decoder;
+	void *audio_output;
 } AndroidChiakiSession;
 
 static JNIEnv *attach_thread_jni()
@@ -238,6 +240,10 @@ JNIEXPORT void JNICALL Java_com_metallic_chiaki_lib_ChiakiNative_sessionCreate(J
 		goto beach;
 	}
 
+	session->audio_output = android_chiaki_audio_output_new(&global_log);
+
+	android_chiaki_audio_decoder_set_cb(&session->audio_decoder, android_chiaki_audio_output_settings, android_chiaki_audio_output_frame, session->audio_output);
+
 	err = chiaki_session_init(&session->session, &connect_info, &global_log);
 	if(err != CHIAKI_ERR_SUCCESS)
 	{
@@ -282,6 +288,7 @@ JNIEXPORT void JNICALL Java_com_metallic_chiaki_lib_ChiakiNative_sessionFree(JNI
 	free(session);
 	android_chiaki_video_decoder_fini(&session->video_decoder);
 	android_chiaki_audio_decoder_fini(&session->audio_decoder);
+	android_chiaki_audio_output_free(session->audio_output);
 	E->DeleteGlobalRef(env, session->java_session);
 }
 

android/app/src/main/cpp/oboe

@@ -0,0 +1 @@
+Subproject commit 0ab5b12a5bc3630a3d6c83b20eed2a669ebf7a24

lib/include/chiaki/thread.h

@@ -107,6 +107,16 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_bool_pred_cond_broadcast(ChiakiBoolPredCond
 
 #ifdef __cplusplus
 }
+
+class ChiakiMutexLock
+{
+	private:
+		ChiakiMutex * const mutex;
+
+	public:
+		ChiakiMutexLock(ChiakiMutex *mutex) : mutex(mutex) { chiaki_mutex_lock(mutex); }
+		~ChiakiMutexLock() { chiaki_mutex_unlock(mutex); }
+};
 #endif
 
 #endif // CHIAKI_THREAD_H