diff --git a/gui/include/streamsession.h b/gui/include/streamsession.h index 3c17ca8..30bf1d7 100644 --- a/gui/include/streamsession.h +++ b/gui/include/streamsession.h @@ -19,6 +19,7 @@ #define CHIAKI_STREAMSESSION_H #include +#include #include "videodecoder.h" #include "exception.h" @@ -65,6 +66,7 @@ class StreamSession : public QObject private: SessionLog log; ChiakiSession session; + ChiakiOpusDecoder opus_decoder; #if CHIAKI_GUI_ENABLE_QT_GAMEPAD QGamepad *gamepad; diff --git a/gui/src/streamsession.cpp b/gui/src/streamsession.cpp index 0c3f0da..6ef8ec6 100644 --- a/gui/src/streamsession.cpp +++ b/gui/src/streamsession.cpp @@ -64,6 +64,8 @@ StreamSession::StreamSession(const StreamSessionConnectInfo &connect_info, QObje audio_output(nullptr), audio_io(nullptr) { + chiaki_opus_decoder_init(&opus_decoder, log.GetChiakiLog()); + QByteArray host_str = connect_info.host.toUtf8(); ChiakiConnectInfo chiaki_connect_info; @@ -84,7 +86,11 @@ StreamSession::StreamSession(const StreamSessionConnectInfo &connect_info, QObje if(err != CHIAKI_ERR_SUCCESS) throw ChiakiException("Chiaki Session Init failed: " + QString::fromLocal8Bit(chiaki_error_string(err))); - chiaki_session_set_audio_cb(&session, AudioSettingsCb, AudioFrameCb, this); + chiaki_opus_decoder_set_cb(&opus_decoder, AudioSettingsCb, AudioFrameCb, this); + ChiakiAudioSink audio_sink; + chiaki_opus_decoder_get_sink(&opus_decoder, &audio_sink); + chiaki_session_set_audio_sink(&session, &audio_sink); + chiaki_session_set_video_sample_cb(&session, VideoSampleCb, this); chiaki_session_set_event_cb(&session, EventCb, this); @@ -102,6 +108,7 @@ StreamSession::~StreamSession() { chiaki_session_join(&session); chiaki_session_fini(&session); + chiaki_opus_decoder_fini(&opus_decoder); #if CHIAKI_GUI_ENABLE_QT_GAMEPAD delete gamepad; #endif diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index ac6eab0..b7930eb 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -32,7 +32,8 @@ set(HEADER_FILES include/chiaki/takionsendbuffer.h include/chiaki/time.h include/chiaki/fec.h - include/chiaki/regist.h) + include/chiaki/regist.h + include/chiaki/opusdecoder.h) set(SOURCE_FILES src/common.c @@ -67,7 +68,8 @@ set(SOURCE_FILES src/takionsendbuffer.c src/time.c src/fec - src/regist.c) + src/regist.c + src/opusdecoder.c) add_subdirectory(protobuf) include_directories("${NANOPB_SOURCE_DIR}") @@ -77,10 +79,12 @@ include_directories("${CHIAKI_LIB_PROTO_INCLUDE_DIR}") if(CHIAKI_LIB_ENABLE_OPUS) find_package(Opus REQUIRED) include_directories(${Opus_INCLUDE_DIRS}) - add_definitions(-DCHIAKI_LIB_ENABLE_OPUS) endif() add_library(chiaki-lib ${HEADER_FILES} ${SOURCE_FILES} ${CHIAKI_LIB_PROTO_SOURCE_FILES} ${CHIAKI_LIB_PROTO_HEADER_FILES}) +configure_file(config.h.in include/chiaki/config.h) +target_include_directories(chiaki-lib PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/include") + add_dependencies(chiaki-lib chiaki-pb) set_target_properties(chiaki-lib PROPERTIES OUTPUT_NAME chiaki) diff --git a/lib/config.h.in b/lib/config.h.in new file mode 100644 index 0000000..d7501e0 --- /dev/null +++ b/lib/config.h.in @@ -0,0 +1,23 @@ +/* + * This file is part of Chiaki. + * + * Chiaki is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chiaki is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Chiaki. If not, see . + */ + +#ifndef CHIAKI_CONFIG_H +#define CHIAKI_CONFIG_H + +#cmakedefine01 CHIAKI_LIB_ENABLE_OPUS + +#endif // CHIAKI_CONFIG_H diff --git a/lib/include/chiaki/audioreceiver.h b/lib/include/chiaki/audioreceiver.h index 43c3ef3..171cabc 100644 --- a/lib/include/chiaki/audioreceiver.h +++ b/lib/include/chiaki/audioreceiver.h @@ -28,18 +28,26 @@ extern "C" { #endif +typedef void (*ChiakiAudioSinkHeader)(ChiakiAudioHeader *header, void *user); +typedef void (*ChiakiAudioSinkFrame)(uint8_t *buf, size_t buf_size, void *user); + +/** + * Sink that receives Audio encoded as Opus + */ +typedef struct chiaki_audio_sink_t +{ + void *user; + ChiakiAudioSinkHeader header_cb; + ChiakiAudioSinkFrame frame_cb; +} ChiakiAudioSink; typedef struct chiaki_audio_receiver_t { struct chiaki_session_t *session; ChiakiLog *log; ChiakiMutex mutex; - struct OpusDecoder *opus_decoder; - ChiakiAudioHeader audio_header; ChiakiSeqNum16 frame_index_prev; bool frame_index_startup; // whether frame_index_prev has definitely not wrapped yet - int16_t *pcm_buf; - size_t pcm_buf_size; } ChiakiAudioReceiver; CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, struct chiaki_session_t *session); diff --git a/lib/include/chiaki/opusdecoder.h b/lib/include/chiaki/opusdecoder.h new file mode 100644 index 0000000..4460b29 --- /dev/null +++ b/lib/include/chiaki/opusdecoder.h @@ -0,0 +1,63 @@ +/* + * This file is part of Chiaki. + * + * Chiaki is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chiaki is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Chiaki. If not, see . + */ + +#ifndef CHIAKI_OPUSDECODER_H +#define CHIAKI_OPUSDECODER_H + +#include +#if CHIAKI_LIB_ENABLE_OPUS + +#include "audioreceiver.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void (*ChiakiOpusDecoderSettingsCallback)(uint32_t channels, uint32_t rate, void *user); +typedef void (*ChiakiOpusDecoderFrameCallback)(int16_t *buf, size_t samples_count, void *user); + +typedef struct chiaki_opus_decoder_t +{ + ChiakiLog *log; + struct OpusDecoder *opus_decoder; + ChiakiAudioHeader audio_header; + int16_t *pcm_buf; + size_t pcm_buf_size; + + ChiakiOpusDecoderSettingsCallback settings_cb; + ChiakiOpusDecoderFrameCallback frame_cb; + void *cb_user; +} ChiakiOpusDecoder; + +CHIAKI_EXPORT void chiaki_opus_decoder_init(ChiakiOpusDecoder *decoder, ChiakiLog *log); +CHIAKI_EXPORT void chiaki_opus_decoder_fini(ChiakiOpusDecoder *decoder); +CHIAKI_EXPORT void chiaki_opus_decoder_get_sink(ChiakiOpusDecoder *decoder, ChiakiAudioSink *sink); + +static inline void chiaki_opus_decoder_set_cb(ChiakiOpusDecoder *decoder, ChiakiOpusDecoderSettingsCallback settings_cb, ChiakiOpusDecoderFrameCallback frame_cb, void *user) +{ + decoder->settings_cb = settings_cb; + decoder->frame_cb = frame_cb; + decoder->cb_user = user; +} + +#ifdef __cplusplus +} +#endif + +#endif + +#endif // CHIAKI_OPUSDECODER_H diff --git a/lib/include/chiaki/session.h b/lib/include/chiaki/session.h index 566988a..45cb596 100644 --- a/lib/include/chiaki/session.h +++ b/lib/include/chiaki/session.h @@ -129,8 +129,6 @@ typedef struct chiaki_event_t } ChiakiEvent; typedef void (*ChiakiEventCallback)(ChiakiEvent *event, void *user); -typedef void (*ChiakiAudioSettingsCallback)(uint32_t channels, uint32_t rate, void *user); -typedef void (*ChiakiAudioFrameCallback)(int16_t *buf, size_t samples_count, void *user); /** * buf will always have an allocated padding of at least CHIAKI_VIDEO_BUFFER_PADDING_SIZE after buf_size @@ -166,11 +164,9 @@ typedef struct chiaki_session_t ChiakiEventCallback event_cb; void *event_cb_user; - ChiakiAudioSettingsCallback audio_settings_cb; - ChiakiAudioFrameCallback audio_frame_cb; - void *audio_cb_user; ChiakiVideoSampleCallback video_sample_cb; void *video_sample_cb_user; + ChiakiAudioSink audio_sink; ChiakiThread session_thread; @@ -210,19 +206,20 @@ static inline void chiaki_session_set_event_cb(ChiakiSession *session, ChiakiEve session->event_cb_user = user; } -static inline void chiaki_session_set_audio_cb(ChiakiSession *session, ChiakiAudioSettingsCallback settings_cb, ChiakiAudioFrameCallback frame_cb, void *user) -{ - session->audio_settings_cb = settings_cb; - session->audio_frame_cb = frame_cb; - session->audio_cb_user = user; -} - static inline void chiaki_session_set_video_sample_cb(ChiakiSession *session, ChiakiVideoSampleCallback cb, void *user) { session->video_sample_cb = cb; session->video_sample_cb_user = user; } +/** + * @param sink contents are copied + */ +static inline void chiaki_session_set_audio_sink(ChiakiSession *session, ChiakiAudioSink *sink) +{ + session->audio_sink = *sink; +} + #ifdef __cplusplus } #endif diff --git a/lib/src/audioreceiver.c b/lib/src/audioreceiver.c index 2a2535d..54f52cd 100644 --- a/lib/src/audioreceiver.c +++ b/lib/src/audioreceiver.c @@ -18,10 +18,6 @@ #include #include -#ifdef CHIAKI_LIB_ENABLE_OPUS -#include -#endif - #include static void chiaki_audio_receiver_frame(ChiakiAudioReceiver *audio_receiver, ChiakiSeqNum16 frame_index, uint8_t *buf, size_t buf_size); @@ -30,15 +26,10 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *au { audio_receiver->session = session; audio_receiver->log = session->log; - audio_receiver->opus_decoder = NULL; - memset(&audio_receiver->audio_header, 0, sizeof(audio_receiver->audio_header)); audio_receiver->frame_index_prev = 0; audio_receiver->frame_index_startup = true; - audio_receiver->pcm_buf = NULL; - audio_receiver->pcm_buf_size = 0; - ChiakiErrorCode err = chiaki_mutex_init(&audio_receiver->mutex, false); if(err != CHIAKI_ERR_SUCCESS) return err; @@ -53,7 +44,6 @@ CHIAKI_EXPORT void chiaki_audio_receiver_fini(ChiakiAudioReceiver *audio_receive opus_decoder_destroy(audio_receiver->opus_decoder); #endif chiaki_mutex_fini(&audio_receiver->mutex); - free(audio_receiver->pcm_buf); } @@ -67,46 +57,10 @@ CHIAKI_EXPORT void chiaki_audio_receiver_stream_info(ChiakiAudioReceiver *audio_ CHIAKI_LOGI(audio_receiver->log, " rate = %d", audio_header->rate); CHIAKI_LOGI(audio_receiver->log, " frame size = %d", audio_header->frame_size); CHIAKI_LOGI(audio_receiver->log, " unknown = %d", audio_header->unknown); - memcpy(&audio_receiver->audio_header, audio_header, sizeof(audio_receiver->audio_header)); -#ifdef CHIAKI_LIB_ENABLE_OPUS - opus_decoder_destroy(audio_receiver->opus_decoder); + if(audio_receiver->session->audio_sink.header_cb) + audio_receiver->session->audio_sink.header_cb(audio_header, audio_receiver->session->audio_sink.user); - int error; - audio_receiver->opus_decoder = opus_decoder_create(audio_header->rate, audio_header->channels, &error); - - if(error != OPUS_OK) - { - CHIAKI_LOGE(audio_receiver->log, "Audio Receiver failed to initialize opus decoder: %s", opus_strerror(error)); - goto beach; - } - - CHIAKI_LOGI(audio_receiver->log, "Audio Receiver initialized opus decoder with the settings above"); - - size_t pcm_buf_size_required = chiaki_audio_header_frame_buf_size(audio_header); - int16_t *pcm_buf_old = audio_receiver->pcm_buf; - if(!audio_receiver->pcm_buf || audio_receiver->pcm_buf_size != pcm_buf_size_required) - audio_receiver->pcm_buf = realloc(audio_receiver->pcm_buf, pcm_buf_size_required); - - if(!audio_receiver->pcm_buf) - { - free(pcm_buf_old); - CHIAKI_LOGE(audio_receiver->log, "Audio Receiver failed to alloc pcm buffer"); - opus_decoder_destroy(audio_receiver->opus_decoder); - audio_receiver->opus_decoder = NULL; - audio_receiver->pcm_buf_size = 0; - goto beach; - } - - audio_receiver->pcm_buf_size = pcm_buf_size_required; - - if(audio_receiver->session->audio_settings_cb) - audio_receiver->session->audio_settings_cb(audio_header->channels, audio_header->rate, audio_receiver->session->audio_cb_user); -#else - CHIAKI_LOGE(audio_receiver->log, "Opus disabled, not initializing decoder"); -#endif - -beach: chiaki_mutex_unlock(&audio_receiver->mutex); } @@ -168,27 +122,15 @@ CHIAKI_EXPORT void chiaki_audio_receiver_av_packet(ChiakiAudioReceiver *audio_re static void chiaki_audio_receiver_frame(ChiakiAudioReceiver *audio_receiver, ChiakiSeqNum16 frame_index, uint8_t *buf, size_t buf_size) { -#ifdef CHIAKI_LIB_ENABLE_OPUS chiaki_mutex_lock(&audio_receiver->mutex); - if(!audio_receiver->opus_decoder) - { - CHIAKI_LOGE(audio_receiver->log, "Received audio frame, but opus decoder is not initialized"); - goto beach; - } - if(!chiaki_seq_num_16_gt(frame_index, audio_receiver->frame_index_prev)) goto beach; - audio_receiver->frame_index_prev = frame_index; - int r = opus_decode(audio_receiver->opus_decoder, buf, (opus_int32)buf_size, audio_receiver->pcm_buf, audio_receiver->audio_header.frame_size, 0); - if(r < 1) - CHIAKI_LOGE(audio_receiver->log, "Decoding audio frame with opus failed: %s", opus_strerror(r)); - else - audio_receiver->session->audio_frame_cb(audio_receiver->pcm_buf, (size_t)r, audio_receiver->session->audio_cb_user); + if(audio_receiver->session->audio_sink.frame_cb) + audio_receiver->session->audio_sink.frame_cb(buf, buf_size, audio_receiver->session->audio_sink.user); beach: chiaki_mutex_unlock(&audio_receiver->mutex); -#endif } \ No newline at end of file diff --git a/lib/src/opusdecoder.c b/lib/src/opusdecoder.c new file mode 100644 index 0000000..ccfa2b0 --- /dev/null +++ b/lib/src/opusdecoder.c @@ -0,0 +1,112 @@ +/* + * This file is part of Chiaki. + * + * Chiaki is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Chiaki is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Chiaki. If not, see . + */ + +#include +#if CHIAKI_LIB_ENABLE_OPUS + +#include + +#include + +#include + +static void chiaki_opus_decoder_header(ChiakiAudioHeader *header, void *user); +static void chiaki_opus_decoder_frame(uint8_t *buf, size_t buf_size, void *user); + +CHIAKI_EXPORT void chiaki_opus_decoder_init(ChiakiOpusDecoder *decoder, ChiakiLog *log) +{ + decoder->log = log; + decoder->opus_decoder = NULL; + memset(&decoder->audio_header, 0, sizeof(decoder->audio_header)); + + decoder->pcm_buf = NULL; + decoder->pcm_buf_size = 0; + + decoder->cb_user = NULL; + decoder->settings_cb = NULL; + decoder->frame_cb = NULL; +} + +CHIAKI_EXPORT void chiaki_opus_decoder_fini(ChiakiOpusDecoder *decoder) +{ + free(decoder->pcm_buf); +} + +CHIAKI_EXPORT void chiaki_opus_decoder_get_sink(ChiakiOpusDecoder *decoder, ChiakiAudioSink *sink) +{ + sink->user = decoder; + sink->header_cb = chiaki_opus_decoder_header; + sink->frame_cb = chiaki_opus_decoder_frame; +} + +static void chiaki_opus_decoder_header(ChiakiAudioHeader *header, void *user) +{ + ChiakiOpusDecoder *decoder = user; + memcpy(&decoder->audio_header, header, sizeof(decoder->audio_header)); + + opus_decoder_destroy(decoder->opus_decoder); + + int error; + decoder->opus_decoder = opus_decoder_create(header->rate, header->channels, &error); + + if(error != OPUS_OK) + { + CHIAKI_LOGE(decoder->log, "ChiakiOpusDecoder failed to initialize opus decoder: %s", opus_strerror(error)); + decoder->opus_decoder = NULL; + return; + } + + CHIAKI_LOGI(decoder->log, "ChiakiOpusDecoder initialized"); + + size_t pcm_buf_size_required = chiaki_audio_header_frame_buf_size(header); + int16_t *pcm_buf_old = decoder->pcm_buf; + if(!decoder->pcm_buf || decoder->pcm_buf_size != pcm_buf_size_required) + decoder->pcm_buf = realloc(decoder->pcm_buf, pcm_buf_size_required); + + if(!decoder->pcm_buf) + { + free(pcm_buf_old); + CHIAKI_LOGE(decoder->log, "ChiakiOpusDecoder failed to alloc pcm buffer"); + opus_decoder_destroy(decoder->opus_decoder); + decoder->opus_decoder = NULL; + decoder->pcm_buf_size = 0; + return; + } + + decoder->pcm_buf_size = pcm_buf_size_required; + + if(decoder->settings_cb) + decoder->settings_cb(header->channels, header->rate, decoder->cb_user); +} + +static void chiaki_opus_decoder_frame(uint8_t *buf, size_t buf_size, void *user) +{ + ChiakiOpusDecoder *decoder = user; + if(!decoder->opus_decoder) + { + CHIAKI_LOGE(decoder->log, "Received audio frame, but opus decoder is not initialized"); + return; + } + + int r = opus_decode(decoder->opus_decoder, buf, (opus_int32)buf_size, decoder->pcm_buf, decoder->audio_header.frame_size, 0); + if(r < 1) + CHIAKI_LOGE(decoder->log, "Decoding audio frame with opus failed: %s", opus_strerror(r)); + else if(decoder->frame_cb) + decoder->frame_cb(decoder->pcm_buf, (size_t)r, decoder->cb_user); +} + +#endif \ No newline at end of file