Refactor Opus Decoding

gui/include/streamsession.h

@@ -19,6 +19,7 @@
 #define CHIAKI_STREAMSESSION_H
 
 #include <chiaki/session.h>
+#include <chiaki/opusdecoder.h>
 
 #include "videodecoder.h"
 #include "exception.h"
@@ -65,6 +66,7 @@ class StreamSession : public QObject
 	private:
 		SessionLog log;
 		ChiakiSession session;
+		ChiakiOpusDecoder opus_decoder;
 
 #if CHIAKI_GUI_ENABLE_QT_GAMEPAD
 		QGamepad *gamepad;

gui/src/streamsession.cpp

@@ -64,6 +64,8 @@ StreamSession::StreamSession(const StreamSessionConnectInfo &connect_info, QObje
 	audio_output(nullptr),
 	audio_io(nullptr)
 {
+	chiaki_opus_decoder_init(&opus_decoder, log.GetChiakiLog());
+
 	QByteArray host_str = connect_info.host.toUtf8();
 
 	ChiakiConnectInfo chiaki_connect_info;
@@ -84,7 +86,11 @@ StreamSession::StreamSession(const StreamSessionConnectInfo &connect_info, QObje
 	if(err != CHIAKI_ERR_SUCCESS)
 		throw ChiakiException("Chiaki Session Init failed: " + QString::fromLocal8Bit(chiaki_error_string(err)));
 
-	chiaki_session_set_audio_cb(&session, AudioSettingsCb, AudioFrameCb, this);
+	chiaki_opus_decoder_set_cb(&opus_decoder, AudioSettingsCb, AudioFrameCb, this);
+	ChiakiAudioSink audio_sink;
+	chiaki_opus_decoder_get_sink(&opus_decoder, &audio_sink);
+	chiaki_session_set_audio_sink(&session, &audio_sink);
+
 	chiaki_session_set_video_sample_cb(&session, VideoSampleCb, this);
 	chiaki_session_set_event_cb(&session, EventCb, this);
 
@@ -102,6 +108,7 @@ StreamSession::~StreamSession()
 {
 	chiaki_session_join(&session);
 	chiaki_session_fini(&session);
+	chiaki_opus_decoder_fini(&opus_decoder);
 #if CHIAKI_GUI_ENABLE_QT_GAMEPAD
 	delete gamepad;
 #endif

lib/CMakeLists.txt

@@ -32,7 +32,8 @@ set(HEADER_FILES
 		include/chiaki/takionsendbuffer.h
 		include/chiaki/time.h
 		include/chiaki/fec.h
-		include/chiaki/regist.h)
+		include/chiaki/regist.h
+		include/chiaki/opusdecoder.h)
 
 set(SOURCE_FILES
 		src/common.c
@@ -67,7 +68,8 @@ set(SOURCE_FILES
 		src/takionsendbuffer.c
 		src/time.c
 		src/fec
-		src/regist.c)
+		src/regist.c
+		src/opusdecoder.c)
 
 add_subdirectory(protobuf)
 include_directories("${NANOPB_SOURCE_DIR}")
@@ -77,10 +79,12 @@ include_directories("${CHIAKI_LIB_PROTO_INCLUDE_DIR}")
 if(CHIAKI_LIB_ENABLE_OPUS)
 	find_package(Opus REQUIRED)
 	include_directories(${Opus_INCLUDE_DIRS})
-	add_definitions(-DCHIAKI_LIB_ENABLE_OPUS)
 endif()
 
 add_library(chiaki-lib ${HEADER_FILES} ${SOURCE_FILES} ${CHIAKI_LIB_PROTO_SOURCE_FILES} ${CHIAKI_LIB_PROTO_HEADER_FILES})
+configure_file(config.h.in include/chiaki/config.h)
+target_include_directories(chiaki-lib PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/include")
+
 add_dependencies(chiaki-lib chiaki-pb)
 set_target_properties(chiaki-lib PROPERTIES OUTPUT_NAME chiaki)
 

lib/config.h.in

@@ -0,0 +1,23 @@
+/*
+ * This file is part of Chiaki.
+ *
+ * Chiaki is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Chiaki is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Chiaki.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#ifndef CHIAKI_CONFIG_H
+#define CHIAKI_CONFIG_H
+
+#cmakedefine01 CHIAKI_LIB_ENABLE_OPUS
+
+#endif // CHIAKI_CONFIG_H

lib/include/chiaki/audioreceiver.h

@@ -28,18 +28,26 @@
 extern "C" {
 #endif
 
+typedef void (*ChiakiAudioSinkHeader)(ChiakiAudioHeader *header, void *user);
+typedef void (*ChiakiAudioSinkFrame)(uint8_t *buf, size_t buf_size, void *user);
+
+/**
+ * Sink that receives Audio encoded as Opus
+ */
+typedef struct chiaki_audio_sink_t
+{
+	void *user;
+	ChiakiAudioSinkHeader header_cb;
+	ChiakiAudioSinkFrame frame_cb;
+} ChiakiAudioSink;
 
 typedef struct chiaki_audio_receiver_t
 {
 	struct chiaki_session_t *session;
 	ChiakiLog *log;
 	ChiakiMutex mutex;
-	struct OpusDecoder *opus_decoder;
-	ChiakiAudioHeader audio_header;
 	ChiakiSeqNum16 frame_index_prev;
 	bool frame_index_startup; // whether frame_index_prev has definitely not wrapped yet
-	int16_t *pcm_buf;
-	size_t pcm_buf_size;
 } ChiakiAudioReceiver;
 
 CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *audio_receiver, struct chiaki_session_t *session);

lib/include/chiaki/opusdecoder.h

@@ -0,0 +1,63 @@
+/*
+ * This file is part of Chiaki.
+ *
+ * Chiaki is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Chiaki is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Chiaki.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#ifndef CHIAKI_OPUSDECODER_H
+#define CHIAKI_OPUSDECODER_H
+
+#include <chiaki/config.h>
+#if CHIAKI_LIB_ENABLE_OPUS
+
+#include "audioreceiver.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void (*ChiakiOpusDecoderSettingsCallback)(uint32_t channels, uint32_t rate, void *user);
+typedef void (*ChiakiOpusDecoderFrameCallback)(int16_t *buf, size_t samples_count, void *user);
+
+typedef struct chiaki_opus_decoder_t
+{
+	ChiakiLog *log;
+	struct OpusDecoder *opus_decoder;
+	ChiakiAudioHeader audio_header;
+	int16_t *pcm_buf;
+	size_t pcm_buf_size;
+
+	ChiakiOpusDecoderSettingsCallback settings_cb;
+	ChiakiOpusDecoderFrameCallback frame_cb;
+	void *cb_user;
+} ChiakiOpusDecoder;
+
+CHIAKI_EXPORT void chiaki_opus_decoder_init(ChiakiOpusDecoder *decoder, ChiakiLog *log);
+CHIAKI_EXPORT void chiaki_opus_decoder_fini(ChiakiOpusDecoder *decoder);
+CHIAKI_EXPORT void chiaki_opus_decoder_get_sink(ChiakiOpusDecoder *decoder, ChiakiAudioSink *sink);
+
+static inline void chiaki_opus_decoder_set_cb(ChiakiOpusDecoder *decoder, ChiakiOpusDecoderSettingsCallback settings_cb, ChiakiOpusDecoderFrameCallback frame_cb, void *user)
+{
+	decoder->settings_cb = settings_cb;
+	decoder->frame_cb = frame_cb;
+	decoder->cb_user = user;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
+#endif // CHIAKI_OPUSDECODER_H

lib/include/chiaki/session.h

@@ -129,8 +129,6 @@ typedef struct chiaki_event_t
 } ChiakiEvent;
 
 typedef void (*ChiakiEventCallback)(ChiakiEvent *event, void *user);
-typedef void (*ChiakiAudioSettingsCallback)(uint32_t channels, uint32_t rate, void *user);
-typedef void (*ChiakiAudioFrameCallback)(int16_t *buf, size_t samples_count, void *user);
 
 /**
  * buf will always have an allocated padding of at least CHIAKI_VIDEO_BUFFER_PADDING_SIZE after buf_size
@@ -166,11 +164,9 @@ typedef struct chiaki_session_t
 
 	ChiakiEventCallback event_cb;
 	void *event_cb_user;
-	ChiakiAudioSettingsCallback audio_settings_cb;
-	ChiakiAudioFrameCallback audio_frame_cb;
-	void *audio_cb_user;
 	ChiakiVideoSampleCallback video_sample_cb;
 	void *video_sample_cb_user;
+	ChiakiAudioSink audio_sink;
 
 	ChiakiThread session_thread;
 
@@ -210,19 +206,20 @@ static inline void chiaki_session_set_event_cb(ChiakiSession *session, ChiakiEve
 	session->event_cb_user = user;
 }
 
-static inline void chiaki_session_set_audio_cb(ChiakiSession *session, ChiakiAudioSettingsCallback settings_cb, ChiakiAudioFrameCallback frame_cb, void *user)
-{
-	session->audio_settings_cb = settings_cb;
-	session->audio_frame_cb = frame_cb;
-	session->audio_cb_user = user;
-}
-
 static inline void chiaki_session_set_video_sample_cb(ChiakiSession *session, ChiakiVideoSampleCallback cb, void *user)
 {
 	session->video_sample_cb = cb;
 	session->video_sample_cb_user = user;
 }
 
+/**
+ * @param sink contents are copied
+ */
+static inline void chiaki_session_set_audio_sink(ChiakiSession *session, ChiakiAudioSink *sink)
+{
+	session->audio_sink = *sink;
+}
+
 #ifdef __cplusplus
 }
 #endif

lib/src/audioreceiver.c

@@ -18,10 +18,6 @@
 #include <chiaki/audioreceiver.h>
 #include <chiaki/session.h>
 
-#ifdef CHIAKI_LIB_ENABLE_OPUS
-#include <opus/opus.h>
-#endif
-
 #include <string.h>
 
 static void chiaki_audio_receiver_frame(ChiakiAudioReceiver *audio_receiver, ChiakiSeqNum16 frame_index, uint8_t *buf, size_t buf_size);
@@ -30,15 +26,10 @@ CHIAKI_EXPORT ChiakiErrorCode chiaki_audio_receiver_init(ChiakiAudioReceiver *au
 {
 	audio_receiver->session = session;
 	audio_receiver->log = session->log;
-	audio_receiver->opus_decoder = NULL;
-	memset(&audio_receiver->audio_header, 0, sizeof(audio_receiver->audio_header));
 
 	audio_receiver->frame_index_prev = 0;
 	audio_receiver->frame_index_startup = true;
 
-	audio_receiver->pcm_buf = NULL;
-	audio_receiver->pcm_buf_size = 0;
-
 	ChiakiErrorCode err = chiaki_mutex_init(&audio_receiver->mutex, false);
 	if(err != CHIAKI_ERR_SUCCESS)
 		return err;
@@ -53,7 +44,6 @@ CHIAKI_EXPORT void chiaki_audio_receiver_fini(ChiakiAudioReceiver *audio_receive
 	opus_decoder_destroy(audio_receiver->opus_decoder);
 #endif
 	chiaki_mutex_fini(&audio_receiver->mutex);
-	free(audio_receiver->pcm_buf);
 }
 
 
@@ -67,46 +57,10 @@ CHIAKI_EXPORT void chiaki_audio_receiver_stream_info(ChiakiAudioReceiver *audio_
 	CHIAKI_LOGI(audio_receiver->log, "  rate = %d", audio_header->rate);
 	CHIAKI_LOGI(audio_receiver->log, "  frame size = %d", audio_header->frame_size);
 	CHIAKI_LOGI(audio_receiver->log, "  unknown = %d", audio_header->unknown);
-	memcpy(&audio_receiver->audio_header, audio_header, sizeof(audio_receiver->audio_header));
 
-#ifdef CHIAKI_LIB_ENABLE_OPUS
+	if(audio_receiver->session->audio_sink.header_cb)
-	opus_decoder_destroy(audio_receiver->opus_decoder);
+		audio_receiver->session->audio_sink.header_cb(audio_header, audio_receiver->session->audio_sink.user);
 
-	int error;
-	audio_receiver->opus_decoder = opus_decoder_create(audio_header->rate, audio_header->channels, &error);
-
-	if(error != OPUS_OK)
-	{
-		CHIAKI_LOGE(audio_receiver->log, "Audio Receiver failed to initialize opus decoder: %s", opus_strerror(error));
-		goto beach;
-	}
-
-	CHIAKI_LOGI(audio_receiver->log, "Audio Receiver initialized opus decoder with the settings above");
-
-	size_t pcm_buf_size_required = chiaki_audio_header_frame_buf_size(audio_header);
-	int16_t *pcm_buf_old = audio_receiver->pcm_buf;
-	if(!audio_receiver->pcm_buf || audio_receiver->pcm_buf_size != pcm_buf_size_required)
-		audio_receiver->pcm_buf = realloc(audio_receiver->pcm_buf, pcm_buf_size_required);
-
-	if(!audio_receiver->pcm_buf)
-	{
-		free(pcm_buf_old);
-		CHIAKI_LOGE(audio_receiver->log, "Audio Receiver failed to alloc pcm buffer");
-		opus_decoder_destroy(audio_receiver->opus_decoder);
-		audio_receiver->opus_decoder = NULL;
-		audio_receiver->pcm_buf_size = 0;
-		goto beach;
-	}
-
-	audio_receiver->pcm_buf_size = pcm_buf_size_required;
-
-	if(audio_receiver->session->audio_settings_cb)
-		audio_receiver->session->audio_settings_cb(audio_header->channels, audio_header->rate, audio_receiver->session->audio_cb_user);
-#else
-	CHIAKI_LOGE(audio_receiver->log, "Opus disabled, not initializing decoder");
-#endif
-
-beach:
 	chiaki_mutex_unlock(&audio_receiver->mutex);
 }
 
@@ -168,27 +122,15 @@ CHIAKI_EXPORT void chiaki_audio_receiver_av_packet(ChiakiAudioReceiver *audio_re
 
 static void chiaki_audio_receiver_frame(ChiakiAudioReceiver *audio_receiver, ChiakiSeqNum16 frame_index, uint8_t *buf, size_t buf_size)
 {
-#ifdef CHIAKI_LIB_ENABLE_OPUS
 	chiaki_mutex_lock(&audio_receiver->mutex);
 
-	if(!audio_receiver->opus_decoder)
-	{
-		CHIAKI_LOGE(audio_receiver->log, "Received audio frame, but opus decoder is not initialized");
-		goto beach;
-	}
-
 	if(!chiaki_seq_num_16_gt(frame_index, audio_receiver->frame_index_prev))
 		goto beach;
-
 	audio_receiver->frame_index_prev = frame_index;
 
-	int r = opus_decode(audio_receiver->opus_decoder, buf, (opus_int32)buf_size, audio_receiver->pcm_buf, audio_receiver->audio_header.frame_size, 0);
+	if(audio_receiver->session->audio_sink.frame_cb)
-	if(r < 1)
+		audio_receiver->session->audio_sink.frame_cb(buf, buf_size, audio_receiver->session->audio_sink.user);
-		CHIAKI_LOGE(audio_receiver->log, "Decoding audio frame with opus failed: %s", opus_strerror(r));
-	else
-		audio_receiver->session->audio_frame_cb(audio_receiver->pcm_buf, (size_t)r, audio_receiver->session->audio_cb_user);
 
 beach:
 	chiaki_mutex_unlock(&audio_receiver->mutex);
-#endif
 }

lib/src/opusdecoder.c

@@ -0,0 +1,112 @@
+/*
+ * This file is part of Chiaki.
+ *
+ * Chiaki is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Chiaki is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Chiaki.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <chiaki/config.h>
+#if CHIAKI_LIB_ENABLE_OPUS
+
+#include <chiaki/opusdecoder.h>
+
+#include <opus/opus.h>
+
+#include <string.h>
+
+static void chiaki_opus_decoder_header(ChiakiAudioHeader *header, void *user);
+static void chiaki_opus_decoder_frame(uint8_t *buf, size_t buf_size, void *user);
+
+CHIAKI_EXPORT void chiaki_opus_decoder_init(ChiakiOpusDecoder *decoder, ChiakiLog *log)
+{
+	decoder->log = log;
+	decoder->opus_decoder = NULL;
+	memset(&decoder->audio_header, 0, sizeof(decoder->audio_header));
+
+	decoder->pcm_buf = NULL;
+	decoder->pcm_buf_size = 0;
+
+	decoder->cb_user = NULL;
+	decoder->settings_cb = NULL;
+	decoder->frame_cb = NULL;
+}
+
+CHIAKI_EXPORT void chiaki_opus_decoder_fini(ChiakiOpusDecoder *decoder)
+{
+	free(decoder->pcm_buf);
+}
+
+CHIAKI_EXPORT void chiaki_opus_decoder_get_sink(ChiakiOpusDecoder *decoder, ChiakiAudioSink *sink)
+{
+	sink->user = decoder;
+	sink->header_cb = chiaki_opus_decoder_header;
+	sink->frame_cb = chiaki_opus_decoder_frame;
+}
+
+static void chiaki_opus_decoder_header(ChiakiAudioHeader *header, void *user)
+{
+	ChiakiOpusDecoder *decoder = user;
+	memcpy(&decoder->audio_header, header, sizeof(decoder->audio_header));
+
+	opus_decoder_destroy(decoder->opus_decoder);
+
+	int error;
+	decoder->opus_decoder = opus_decoder_create(header->rate, header->channels, &error);
+
+	if(error != OPUS_OK)
+	{
+		CHIAKI_LOGE(decoder->log, "ChiakiOpusDecoder failed to initialize opus decoder: %s", opus_strerror(error));
+		decoder->opus_decoder = NULL;
+		return;
+	}
+
+	CHIAKI_LOGI(decoder->log, "ChiakiOpusDecoder initialized");
+
+	size_t pcm_buf_size_required = chiaki_audio_header_frame_buf_size(header);
+	int16_t *pcm_buf_old = decoder->pcm_buf;
+	if(!decoder->pcm_buf || decoder->pcm_buf_size != pcm_buf_size_required)
+		decoder->pcm_buf = realloc(decoder->pcm_buf, pcm_buf_size_required);
+
+	if(!decoder->pcm_buf)
+	{
+		free(pcm_buf_old);
+		CHIAKI_LOGE(decoder->log, "ChiakiOpusDecoder failed to alloc pcm buffer");
+		opus_decoder_destroy(decoder->opus_decoder);
+		decoder->opus_decoder = NULL;
+		decoder->pcm_buf_size = 0;
+		return;
+	}
+
+	decoder->pcm_buf_size = pcm_buf_size_required;
+
+	if(decoder->settings_cb)
+		decoder->settings_cb(header->channels, header->rate, decoder->cb_user);
+}
+
+static void chiaki_opus_decoder_frame(uint8_t *buf, size_t buf_size, void *user)
+{
+	ChiakiOpusDecoder *decoder = user;
+	if(!decoder->opus_decoder)
+	{
+		CHIAKI_LOGE(decoder->log, "Received audio frame, but opus decoder is not initialized");
+		return;
+	}
+
+	int r = opus_decode(decoder->opus_decoder, buf, (opus_int32)buf_size, decoder->pcm_buf, decoder->audio_header.frame_size, 0);
+	if(r < 1)
+		CHIAKI_LOGE(decoder->log, "Decoding audio frame with opus failed: %s", opus_strerror(r));
+	else if(decoder->frame_cb)
+		decoder->frame_cb(decoder->pcm_buf, (size_t)r, decoder->cb_user);
+}
+
+#endif