diff --git a/android/app/CMakeLists.txt b/android/app/CMakeLists.txt index bc02212..10fc8f4 100644 --- a/android/app/CMakeLists.txt +++ b/android/app/CMakeLists.txt @@ -4,7 +4,9 @@ cmake_minimum_required(VERSION 3.2) add_library(chiaki-jni SHARED src/main/cpp/chiaki-jni.c src/main/cpp/video-decoder.h - src/main/cpp/video-decoder.c) + src/main/cpp/video-decoder.c + src/main/cpp/audio-decoder.h + src/main/cpp/audio-decoder.c) target_link_libraries(chiaki-jni chiaki-lib) find_library(ANDROID_LIB_LOG log) diff --git a/android/app/src/main/cpp/audio-decoder.c b/android/app/src/main/cpp/audio-decoder.c new file mode 100644 index 0000000..ad46809 --- /dev/null +++ b/android/app/src/main/cpp/audio-decoder.c @@ -0,0 +1,179 @@ +/* + * This file is part of Chiaki. + * + * Chiaki is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chiaki is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Chiaki. If not, see . + */ + +#include "audio-decoder.h" + +#include + +#include +#include +#include + +#include + +#define INPUT_BUFFER_TIMEOUT_MS 10 + +static void *android_chiaki_audio_decoder_output_thread_func(void *user); +static void android_chiaki_audio_decoder_header(ChiakiAudioHeader *header, void *user); +static void android_chiaki_audio_decoder_frame(uint8_t *buf, size_t buf_size, void *user); + +ChiakiErrorCode android_chiaki_audio_decoder_init(AndroidChiakiAudioDecoder *decoder, ChiakiLog *log) +{ + decoder->log = log; + memset(&decoder->audio_header, 0, sizeof(decoder->audio_header)); + decoder->codec = NULL; + decoder->timestamp_cur = 0; + return CHIAKI_ERR_SUCCESS; +} + +void android_chiaki_audio_decoder_fini(AndroidChiakiAudioDecoder *decoder) +{ + // TODO: shutdown (thread may or may not be running!) +} + +void android_chiaki_audio_decoder_get_sink(AndroidChiakiAudioDecoder *decoder, ChiakiAudioSink *sink) +{ + sink->user = decoder; + sink->header_cb = android_chiaki_audio_decoder_header; + sink->frame_cb = android_chiaki_audio_decoder_frame; +} + +static void *android_chiaki_audio_decoder_output_thread_func(void *user) +{ + AndroidChiakiAudioDecoder *decoder = user; + + while(1) + { + AMediaCodecBufferInfo info; + ssize_t status = AMediaCodec_dequeueOutputBuffer(decoder->codec, &info, -1); + if(status >= 0) + { + CHIAKI_LOGD(decoder->log, "Got decoded audio of size %d", (int)info.size); + AMediaCodec_releaseOutputBuffer(decoder->codec, (size_t)status, info.size != 0); + if(info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) + { + CHIAKI_LOGI(decoder->log, "AMediaCodec for Audio Decoder reported EOS"); + break; + } + } + } + + return NULL; +} + +static void android_chiaki_audio_decoder_header(ChiakiAudioHeader *header, void *user) +{ + AndroidChiakiAudioDecoder *decoder = user; + memcpy(&decoder->audio_header, header, sizeof(decoder->audio_header)); + + if(decoder->codec) + { + CHIAKI_LOGI(decoder->log, "Audio decoder already initialized, re-creating"); + // TODO: stop thread + AMediaCodec_delete(decoder->codec); + decoder->codec = NULL; + } + + const char *mime = "audio/opus"; + decoder->codec = AMediaCodec_createDecoderByType(mime); + if(!decoder->codec) + { + CHIAKI_LOGE(decoder->log, "Failed to create AMediaCodec for mime type %s", mime); + return; + } + + AMediaFormat *format = AMediaFormat_new(); + AMediaFormat_setString(format, AMEDIAFORMAT_KEY_MIME, mime); + AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_CHANNEL_COUNT, header->channels); + AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_SAMPLE_RATE, header->rate); + + AMediaCodec_configure(decoder->codec, format, NULL, NULL, 0); // TODO: check result + AMediaCodec_start(decoder->codec); // TODO: check result + + AMediaFormat_delete(format); + + ChiakiErrorCode err = chiaki_thread_create(&decoder->output_thread, android_chiaki_audio_decoder_output_thread_func, decoder); + if(err != CHIAKI_ERR_SUCCESS) + { + CHIAKI_LOGE(decoder->log, "Failed to create output thread for AMediaCodec"); + AMediaCodec_delete(decoder->codec); + decoder->codec = NULL; + } + + + uint8_t opus_id_head[0x13]; + memcpy(opus_id_head, "OpusHead", 8); + opus_id_head[0x8] = 1; // version + opus_id_head[0x9] = header->channels; + uint16_t pre_skip = 3840; + opus_id_head[0xa] = (uint8_t)(pre_skip & 0xff); + opus_id_head[0xb] = (uint8_t)(pre_skip >> 8); + opus_id_head[0xc] = (uint8_t)(header->rate & 0xff); + opus_id_head[0xd] = (uint8_t)((header->rate >> 0x8) & 0xff); + opus_id_head[0xe] = (uint8_t)((header->rate >> 0x10) & 0xff); + opus_id_head[0xf] = (uint8_t)(header->rate >> 0x18); + uint16_t output_gain = 0; + opus_id_head[0x10] = (uint8_t)(output_gain & 0xff); + opus_id_head[0x11] = (uint8_t)(output_gain >> 8); + opus_id_head[0x12] = 0; // channel map + //AMediaFormat_setBuffer(format, AMEDIAFORMAT_KEY_CSD_0, opus_id_head, sizeof(opus_id_head)); + android_chiaki_audio_decoder_frame(opus_id_head, sizeof(opus_id_head), decoder); + + uint64_t pre_skip_ns = 0; + uint8_t csd1[8] = { (uint8_t)(pre_skip_ns & 0xff), (uint8_t)((pre_skip_ns >> 0x8) & 0xff), (uint8_t)((pre_skip_ns >> 0x10) & 0xff), (uint8_t)((pre_skip_ns >> 0x18) & 0xff), + (uint8_t)((pre_skip_ns >> 0x20) & 0xff), (uint8_t)((pre_skip_ns >> 0x28) & 0xff), (uint8_t)((pre_skip_ns >> 0x30) & 0xff), (uint8_t)(pre_skip_ns >> 0x38)}; + android_chiaki_audio_decoder_frame(csd1, sizeof(csd1), decoder); + + uint64_t pre_roll_ns = 0; + uint8_t csd2[8] = { (uint8_t)(pre_roll_ns & 0xff), (uint8_t)((pre_roll_ns >> 0x8) & 0xff), (uint8_t)((pre_roll_ns >> 0x10) & 0xff), (uint8_t)((pre_roll_ns >> 0x18) & 0xff), + (uint8_t)((pre_roll_ns >> 0x20) & 0xff), (uint8_t)((pre_roll_ns >> 0x28) & 0xff), (uint8_t)((pre_roll_ns >> 0x30) & 0xff), (uint8_t)(pre_roll_ns >> 0x38)}; + android_chiaki_audio_decoder_frame(csd2, sizeof(csd2), decoder); +} + +static void android_chiaki_audio_decoder_frame(uint8_t *buf, size_t buf_size, void *user) +{ + AndroidChiakiAudioDecoder *decoder = user; + + if(!decoder->codec) + { + CHIAKI_LOGE(decoder->log, "Received audio data, but decoder is not initialized!"); + return; + } + + while(buf_size > 0) + { + ssize_t codec_buf_index = AMediaCodec_dequeueInputBuffer(decoder->codec, INPUT_BUFFER_TIMEOUT_MS * 1000); + if(codec_buf_index < 0) + { + CHIAKI_LOGE(decoder->log, "Failed to get input audio buffer"); + return; + } + + size_t codec_buf_size; + uint8_t *codec_buf = AMediaCodec_getInputBuffer(decoder->codec, (size_t)codec_buf_index, &codec_buf_size); + size_t codec_sample_size = buf_size; + if(codec_sample_size > codec_buf_size) + { + CHIAKI_LOGD(decoder->log, "Sample is bigger than audio buffer, splitting"); + codec_sample_size = codec_buf_size; + } + memcpy(codec_buf, buf, codec_sample_size); + AMediaCodec_queueInputBuffer(decoder->codec, (size_t)codec_buf_index, 0, codec_sample_size, decoder->timestamp_cur++, 0); // timestamp just raised by 1 for maximum realtime + buf += codec_sample_size; + buf_size -= codec_sample_size; + } +} \ No newline at end of file diff --git a/android/app/src/main/cpp/audio-decoder.h b/android/app/src/main/cpp/audio-decoder.h new file mode 100644 index 0000000..317226f --- /dev/null +++ b/android/app/src/main/cpp/audio-decoder.h @@ -0,0 +1,40 @@ +/* + * This file is part of Chiaki. + * + * Chiaki is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chiaki is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Chiaki. If not, see . + */ + +#ifndef CHIAKI_JNI_AUDIO_DECODER_H +#define CHIAKI_JNI_AUDIO_DECODER_H + +#include + +#include +#include +#include + +typedef struct android_chiaki_audio_decoder_t +{ + ChiakiLog *log; + ChiakiAudioHeader audio_header; + struct AMediaCodec *codec; + uint64_t timestamp_cur; + ChiakiThread output_thread; +} AndroidChiakiAudioDecoder; + +ChiakiErrorCode android_chiaki_audio_decoder_init(AndroidChiakiAudioDecoder *decoder, ChiakiLog *log); +void android_chiaki_audio_decoder_fini(AndroidChiakiAudioDecoder *decoder); +void android_chiaki_audio_decoder_get_sink(AndroidChiakiAudioDecoder *decoder, ChiakiAudioSink *sink); + +#endif \ No newline at end of file diff --git a/android/app/src/main/cpp/chiaki-jni.c b/android/app/src/main/cpp/chiaki-jni.c index ad448e3..ee51b36 100644 --- a/android/app/src/main/cpp/chiaki-jni.c +++ b/android/app/src/main/cpp/chiaki-jni.c @@ -26,6 +26,7 @@ #include #include "video-decoder.h" +#include "audio-decoder.h" #define LOG_TAG "Chiaki" #define JNI_VERSION JNI_VERSION_1_6 @@ -113,6 +114,7 @@ typedef struct android_chiaki_session_t jfieldID java_controller_state_right_y; AndroidChiakiVideoDecoder video_decoder; + AndroidChiakiAudioDecoder audio_decoder; } AndroidChiakiSession; static JNIEnv *attach_thread_jni() @@ -227,9 +229,21 @@ JNIEXPORT void JNICALL Java_com_metallic_chiaki_lib_ChiakiNative_sessionCreate(J goto beach; } + err = android_chiaki_audio_decoder_init(&session->audio_decoder, &global_log); + if(err != CHIAKI_ERR_SUCCESS) + { + android_chiaki_video_decoder_fini(&session->video_decoder); + free(session); + session = NULL; + goto beach; + } + err = chiaki_session_init(&session->session, &connect_info, &global_log); if(err != CHIAKI_ERR_SUCCESS) + { + // TODO: free audio and video decoders and session (?) goto beach; + } session->java_session = E->NewGlobalRef(env, java_session); session->java_session_class = E->GetObjectClass(env, session->java_session); @@ -248,6 +262,10 @@ JNIEXPORT void JNICALL Java_com_metallic_chiaki_lib_ChiakiNative_sessionCreate(J chiaki_session_set_event_cb(&session->session, android_chiaki_event_cb, session); chiaki_session_set_video_sample_cb(&session->session, android_chiaki_video_decoder_video_sample, &session->video_decoder); + ChiakiAudioSink audio_sink; + android_chiaki_audio_decoder_get_sink(&session->audio_decoder, &audio_sink); + chiaki_session_set_audio_sink(&session->session, &audio_sink); + beach: free(host_str); E->SetIntField(env, result, E->GetFieldID(env, result_class, "errorCode", "I"), (jint)err); @@ -263,6 +281,7 @@ JNIEXPORT void JNICALL Java_com_metallic_chiaki_lib_ChiakiNative_sessionFree(JNI chiaki_session_fini(&session->session); free(session); android_chiaki_video_decoder_fini(&session->video_decoder); + android_chiaki_audio_decoder_fini(&session->audio_decoder); E->DeleteGlobalRef(env, session->java_session); } diff --git a/android/app/src/main/cpp/video-decoder.c b/android/app/src/main/cpp/video-decoder.c index 505df31..b2a0437 100644 --- a/android/app/src/main/cpp/video-decoder.c +++ b/android/app/src/main/cpp/video-decoder.c @@ -39,6 +39,7 @@ ChiakiErrorCode android_chiaki_video_decoder_init(AndroidChiakiVideoDecoder *dec void android_chiaki_video_decoder_fini(AndroidChiakiVideoDecoder *decoder) { + // TODO: shutdown (thread may or may not be running!) chiaki_mutex_fini(&decoder->mutex); } @@ -75,8 +76,8 @@ void android_chiaki_video_decoder_set_surface(AndroidChiakiVideoDecoder *decoder AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_WIDTH, 1280); // TODO: correct values AMediaFormat_setInt32(format, AMEDIAFORMAT_KEY_HEIGHT, 720); - AMediaCodec_configure(decoder->codec, format, decoder->window, NULL, 0); - AMediaCodec_start(decoder->codec); + AMediaCodec_configure(decoder->codec, format, decoder->window, NULL, 0); // TODO: check result + AMediaCodec_start(decoder->codec); // TODO: check result AMediaFormat_delete(format); diff --git a/lib/src/streamconnection.c b/lib/src/streamconnection.c index 58ada6e..8b3e404 100644 --- a/lib/src/streamconnection.c +++ b/lib/src/streamconnection.c @@ -608,6 +608,9 @@ static void stream_connection_takion_data_expect_streaminfo(ChiakiStreamConnecti return; } + CHIAKI_LOGD(stream_connection->log, "StreamConnection received audio header:"); + chiaki_log_hexdump(stream_connection->log, CHIAKI_LOG_DEBUG, audio_header, audio_header_buf.size); + if(audio_header_buf.size != CHIAKI_AUDIO_HEADER_SIZE) { CHIAKI_LOGE(stream_connection->log, "StreamConnection received invalid audio header in streaminfo");